dev/null |binary search-server/Makefile | 2 search-server/README | 17 search-server/build.properties | 22 search-server/build.xml | 241 search-server/buildconf/LICENSE.txt | 14 search-server/buildconf/build-utils.xml | 19 search-server/buildconf/exclude | 2 search-server/checkstyle.xml | 233 search-server/ivy.xml | 28 search-server/nutch/crawl_jsp.sh | 35 search-server/nutch/crawl_jsp/conf/automaton-urlfilter.txt | 34 search-server/nutch/crawl_jsp/conf/common-terms.utf8 | 28 search-server/nutch/crawl_jsp/conf/configuration.xsl | 40 search-server/nutch/crawl_jsp/conf/context.xsl | 88 search-server/nutch/crawl_jsp/conf/crawl-tool.xml | 59 search-server/nutch/crawl_jsp/conf/crawl-urlfilter.txt | 61 search-server/nutch/crawl_jsp/conf/domain-suffixes.xml | 4354 ---------- search-server/nutch/crawl_jsp/conf/domain-suffixes.xsd | 130 search-server/nutch/crawl_jsp/conf/hadoop-env.sh | 33 search-server/nutch/crawl_jsp/conf/hadoop-site.xml | 8 search-server/nutch/crawl_jsp/conf/httpclient-auth.xml | 61 search-server/nutch/crawl_jsp/conf/log4j.properties | 82 search-server/nutch/crawl_jsp/conf/logging.properties | 4 search-server/nutch/crawl_jsp/conf/masters | 1 search-server/nutch/crawl_jsp/conf/nutch-conf.xsl | 24 search-server/nutch/crawl_jsp/conf/nutch-default.xml | 1205 -- search-server/nutch/crawl_jsp/conf/nutch-site.xml | 58 search-server/nutch/crawl_jsp/conf/parse-plugins.dtd | 12 search-server/nutch/crawl_jsp/conf/parse-plugins.xml | 221 search-server/nutch/crawl_jsp/conf/prefix-urlfilter.txt | 21 search-server/nutch/crawl_jsp/conf/regex-normalize.xml | 66 search-server/nutch/crawl_jsp/conf/regex-urlfilter.txt | 38 search-server/nutch/crawl_jsp/conf/slaves | 1 search-server/nutch/crawl_jsp/conf/subcollections.xml | 28 search-server/nutch/crawl_jsp/conf/suffix-urlfilter.txt | 91 search-server/nutch/crawl_jsp/conf/tika-mimetypes.xml | 367 search-server/nutch/crawl_jsp/urls/urls.txt | 7 search-server/rhn_search_daemon_dev.conf | 45 search-server/run.sh | 17 search-server/scripts/Reindex.groovy | 43 search-server/scripts/build.properties | 22 search-server/scripts/build.xml | 256 search-server/scripts/buildconf/LICENSE.txt | 14 search-server/scripts/buildconf/build-utils.xml | 19 search-server/scripts/checkstyle.xml | 233 search-server/scripts/ivy.xml | 28 search-server/scripts/maketar.sh | 30 search-server/scripts/reindex.sh | 15 search-server/scripts/search.admin.updateIndex.test.py | 15 search-server/scripts/search.py | 69 search-server/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java | 304 search-server/scripts/webCrawl.sh | 17 search-server/spacewalk-search.spec | 255 search-server/spacewalk-search/Makefile | 2 search-server/spacewalk-search/README | 17 search-server/spacewalk-search/build.properties | 22 search-server/spacewalk-search/build.xml | 241 search-server/spacewalk-search/buildconf/LICENSE.txt | 14 search-server/spacewalk-search/buildconf/build-utils.xml | 19 search-server/spacewalk-search/buildconf/exclude | 2 search-server/spacewalk-search/buildconf/tempjars/hadoop-0.18.1-core.jar |binary search-server/spacewalk-search/buildconf/tempjars/ibatis-2.3.0.677.jar |binary search-server/spacewalk-search/buildconf/tempjars/lucene-analyzers-2.3.2.jar |binary search-server/spacewalk-search/buildconf/tempjars/lucene-core-2.3.2.jar |binary search-server/spacewalk-search/buildconf/tempjars/nutch-2008-12-01_04-01-21.jar |binary search-server/spacewalk-search/buildconf/tempjars/picocontainer-1.3.jar |binary search-server/spacewalk-search/buildconf/tempjars/simple-core-3.1.3.jar |binary search-server/spacewalk-search/checkstyle.xml | 233 search-server/spacewalk-search/ivy.xml | 28 search-server/spacewalk-search/libsrc/hadoop-0.16.4-src.jar |binary search-server/spacewalk-search/libsrc/ibatis-2.3.0.677-src.zip |binary search-server/spacewalk-search/libsrc/icu4j-3_8_1-src.jar |binary search-server/spacewalk-search/libsrc/log4j-1.2.8-src.jar |binary search-server/spacewalk-search/libsrc/lucene-2.3.2-src.zip |binary search-server/spacewalk-search/libsrc/nutch-1.0dev_2008_05_14-src.jar |binary search-server/spacewalk-search/libsrc/nutch-2008-12-01_04-01-21-src.zip |binary search-server/spacewalk-search/libsrc/quartz-1.5.1-src.jar |binary search-server/spacewalk-search/libsrc/redstone-xmlrpc-1.1_20071120-src.jar |binary search-server/spacewalk-search/libsrc/simple-3.1.3-src.zip |binary search-server/spacewalk-search/libsrc/tanukiwrapper-3.1.2-src.jar |binary search-server/spacewalk-search/nutch/crawl_jsp.sh | 35 search-server/spacewalk-search/nutch/crawl_jsp/conf/automaton-urlfilter.txt | 34 search-server/spacewalk-search/nutch/crawl_jsp/conf/common-terms.utf8 | 28 search-server/spacewalk-search/nutch/crawl_jsp/conf/configuration.xsl | 40 search-server/spacewalk-search/nutch/crawl_jsp/conf/context.xsl | 88 search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-tool.xml | 59 search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-urlfilter.txt | 61 search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xml | 4354 ++++++++++ search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xsd | 130 search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-env.sh | 33 search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-site.xml | 8 search-server/spacewalk-search/nutch/crawl_jsp/conf/httpclient-auth.xml | 61 search-server/spacewalk-search/nutch/crawl_jsp/conf/log4j.properties | 82 search-server/spacewalk-search/nutch/crawl_jsp/conf/logging.properties | 4 search-server/spacewalk-search/nutch/crawl_jsp/conf/masters | 1 search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-conf.xsl | 24 search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-default.xml | 1205 ++ search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-site.xml | 58 search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.dtd | 12 search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.xml | 221 search-server/spacewalk-search/nutch/crawl_jsp/conf/prefix-urlfilter.txt | 21 search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-normalize.xml | 66 search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-urlfilter.txt | 38 search-server/spacewalk-search/nutch/crawl_jsp/conf/slaves | 1 search-server/spacewalk-search/nutch/crawl_jsp/conf/subcollections.xml | 28 search-server/spacewalk-search/nutch/crawl_jsp/conf/suffix-urlfilter.txt | 91 search-server/spacewalk-search/nutch/crawl_jsp/conf/tika-mimetypes.xml | 367 search-server/spacewalk-search/nutch/crawl_jsp/urls/urls.txt | 7 search-server/spacewalk-search/rhn_search_daemon_dev.conf | 45 search-server/spacewalk-search/run.sh | 17 search-server/spacewalk-search/scripts/Reindex.groovy | 43 search-server/spacewalk-search/scripts/build.properties | 22 search-server/spacewalk-search/scripts/build.xml | 256 search-server/spacewalk-search/scripts/buildconf/LICENSE.txt | 14 search-server/spacewalk-search/scripts/buildconf/build-utils.xml | 19 search-server/spacewalk-search/scripts/checkstyle.xml | 233 search-server/spacewalk-search/scripts/ivy.xml | 28 search-server/spacewalk-search/scripts/lukeall-0.8.1.jar |binary search-server/spacewalk-search/scripts/maketar.sh | 30 search-server/spacewalk-search/scripts/reindex.sh | 15 search-server/spacewalk-search/scripts/search.admin.updateIndex.test.py | 15 search-server/spacewalk-search/scripts/search.py | 69 search-server/spacewalk-search/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java | 304 search-server/spacewalk-search/scripts/webCrawl.sh | 17 search-server/spacewalk-search/spacewalk-search.spec | 255 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/config.xml | 39 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata.xml | 67 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata_handler.xml | 49 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/hwdevice.xml | 53 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/package.xml | 85 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server.xml | 105 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server_custom_info.xml | 48 search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/snapshot_tag.xml | 53 search-server/spacewalk-search/src/config/etc/logrotate.d/rhn-search | 9 search-server/spacewalk-search/src/config/log4j.properties | 35 search-server/spacewalk-search/src/config/log4j.test.properties | 17 search-server/spacewalk-search/src/config/rhn-search | 395 search-server/spacewalk-search/src/config/search/rhn_search.conf | 15 search-server/spacewalk-search/src/config/search/rhn_search_daemon.conf | 86 search-server/spacewalk-search/src/config/test.properties | 5 search-server/spacewalk-search/src/java/com/redhat/satellite/search/ContainerRunner.java | 62 search-server/spacewalk-search/src/java/com/redhat/satellite/search/DeleteIndexes.java | 187 search-server/spacewalk-search/src/java/com/redhat/satellite/search/Main.java | 89 search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/ConfigException.java | 49 search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/Configuration.java | 622 + search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/KeyTranslator.java | 39 search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java | 137 search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java | 49 search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java | 60 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Connection.java | 97 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/DatabaseManager.java | 94 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Query.java | 95 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/ResultHandler.java | 35 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/WriteQuery.java | 83 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Errata.java | 323 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/GenericRecord.java | 35 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java | 183 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/RhnPackage.java | 176 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Server.java | 698 + search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java | 146 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java | 128 search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java | 45 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/DocResult.java | 119 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java | 71 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexManager.java | 837 + search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexingException.java | 66 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java | 48 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/MatchingField.java | 119 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/QueryParseException.java | 54 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/Result.java | 152 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java | 86 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java | 89 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java | 67 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java | 36 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java | 60 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java | 59 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java | 59 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java | 64 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java | 71 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java | 64 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java | 68 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java | 82 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java | 49 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java | 57 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java | 50 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java | 51 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java | 48 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java | 70 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java | 76 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java | 149 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java | 49 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java | 210 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java | 46 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java | 163 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java | 93 search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java | 26 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/RpcServer.java | 159 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java | 96 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java | 58 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java | 177 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java | 221 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java | 54 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java | 61 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java | 62 search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java | 19 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java | 202 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java | 286 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java | 140 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java | 178 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java | 117 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java | 163 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java | 167 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java | 110 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java | 155 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java | 185 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java | 186 search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java | 48 search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/BaseTestCase.java | 33 search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/TestUtil.java | 133 search-server/spacewalk-search/task-lib/ant-contrib-1.0.jar |binary search-server/spacewalk-search/task-lib/antlr-2.7.6.jar |binary search-server/spacewalk-search/task-lib/checkstyle-4.1.jar |binary search-server/spacewalk-search/task-lib/commons-beanutils-1.7.0.jar |binary search-server/spacewalk-search/task-lib/commons-logging-1.0.4.jar |binary search-server/spacewalk-search/task-lib/emma-20040713.jar |binary search-server/spacewalk-search/task-lib/emma_ant-20040713.jar |binary search-server/spacewalk-search/task-lib/ivy.jar |binary search-server/spacewalk-search/task-lib/junit-3.8.1.jar |binary search-server/spacewalk-search/task-lib/regexp-1.3.jar |binary search-server/src/config/com/redhat/satellite/search/db/config.xml | 39 search-server/src/config/com/redhat/satellite/search/db/errata.xml | 67 search-server/src/config/com/redhat/satellite/search/db/errata_handler.xml | 49 search-server/src/config/com/redhat/satellite/search/db/hwdevice.xml | 53 search-server/src/config/com/redhat/satellite/search/db/package.xml | 85 search-server/src/config/com/redhat/satellite/search/db/server.xml | 105 search-server/src/config/com/redhat/satellite/search/db/server_custom_info.xml | 48 search-server/src/config/com/redhat/satellite/search/db/snapshot_tag.xml | 53 search-server/src/config/etc/logrotate.d/rhn-search | 9 search-server/src/config/log4j.properties | 35 search-server/src/config/log4j.test.properties | 17 search-server/src/config/rhn-search | 395 search-server/src/config/search/rhn_search.conf | 15 search-server/src/config/search/rhn_search_daemon.conf | 86 search-server/src/config/test.properties | 5 search-server/src/java/com/redhat/satellite/search/ContainerRunner.java | 62 search-server/src/java/com/redhat/satellite/search/DeleteIndexes.java | 187 search-server/src/java/com/redhat/satellite/search/Main.java | 89 search-server/src/java/com/redhat/satellite/search/config/ConfigException.java | 49 search-server/src/java/com/redhat/satellite/search/config/Configuration.java | 622 - search-server/src/java/com/redhat/satellite/search/config/KeyTranslator.java | 39 search-server/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java | 137 search-server/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java | 49 search-server/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java | 60 search-server/src/java/com/redhat/satellite/search/db/Connection.java | 97 search-server/src/java/com/redhat/satellite/search/db/DatabaseManager.java | 94 search-server/src/java/com/redhat/satellite/search/db/Query.java | 95 search-server/src/java/com/redhat/satellite/search/db/ResultHandler.java | 35 search-server/src/java/com/redhat/satellite/search/db/WriteQuery.java | 83 search-server/src/java/com/redhat/satellite/search/db/models/Errata.java | 323 search-server/src/java/com/redhat/satellite/search/db/models/GenericRecord.java | 35 search-server/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java | 183 search-server/src/java/com/redhat/satellite/search/db/models/RhnPackage.java | 176 search-server/src/java/com/redhat/satellite/search/db/models/Server.java | 698 - search-server/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java | 146 search-server/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java | 128 search-server/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java | 45 search-server/src/java/com/redhat/satellite/search/index/DocResult.java | 119 search-server/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java | 71 search-server/src/java/com/redhat/satellite/search/index/IndexManager.java | 837 - search-server/src/java/com/redhat/satellite/search/index/IndexingException.java | 66 search-server/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java | 48 search-server/src/java/com/redhat/satellite/search/index/MatchingField.java | 119 search-server/src/java/com/redhat/satellite/search/index/QueryParseException.java | 54 search-server/src/java/com/redhat/satellite/search/index/Result.java | 152 search-server/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java | 86 search-server/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java | 89 search-server/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java | 67 search-server/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java | 36 search-server/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java | 60 search-server/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java | 59 search-server/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java | 59 search-server/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java | 64 search-server/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java | 71 search-server/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java | 64 search-server/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java | 68 search-server/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java | 82 search-server/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java | 49 search-server/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java | 57 search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java | 50 search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java | 51 search-server/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java | 48 search-server/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java | 70 search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java | 76 search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java | 149 search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java | 49 search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java | 210 search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java | 46 search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java | 163 search-server/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java | 93 search-server/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java | 26 search-server/src/java/com/redhat/satellite/search/rpc/RpcServer.java | 159 search-server/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java | 96 search-server/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java | 58 search-server/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java | 177 search-server/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java | 221 search-server/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java | 54 search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java | 61 search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java | 62 search-server/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java | 19 search-server/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java | 202 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java | 286 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java | 140 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java | 178 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java | 117 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java | 163 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java | 167 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java | 110 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java | 155 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java | 185 search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java | 186 search-server/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java | 48 search-server/src/java/com/redhat/satellite/search/tests/BaseTestCase.java | 33 search-server/src/java/com/redhat/satellite/search/tests/TestUtil.java | 133 324 files changed, 20028 insertions(+), 20028 deletions(-)
New commits: commit de91ecc6876c661298116975ec70b66d034b29c4 Author: Miroslav SuchÜ msuchy@redhat.com Date: Wed Jan 27 13:54:18 2010 +0100
move spacewalk-search server to its own directory so we do not pack to spacewalk-search.tar.gz content of spacewalk-doc-indexes
diff --git a/search-server/Makefile b/search-server/Makefile deleted file mode 100644 index 700fc9e..0000000 --- a/search-server/Makefile +++ /dev/null @@ -1,2 +0,0 @@ - -include ../rel-eng/Makefile diff --git a/search-server/README b/search-server/README deleted file mode 100644 index b6f3d23..0000000 --- a/search-server/README +++ /dev/null @@ -1,17 +0,0 @@ -search-server is the new server used to search packages, and ultimately all -content including systems, errata, and documentation. - -build.properties - build configuration -build.xml - master ant build script -buildconf/exclude - list of files excluded from compilation (temp solution) -buildconf/build-utils.xml - ant macro definitions -checkstyle.xml - checkstyle code style definition -ivy.xml - ivy repo configuration file -libsrc - src code for the libraries used -nutch - configuration files for nutch crawler (not currently used) -spacewalk-search.spec - search server RPM spec file -rhn_search_daemon_dev.conf - development based configuration file -run.sh - script used to run the software, especially during development -scripts - useful scripts -src - the source code -task-lib - the libraries used by the search-server diff --git a/search-server/build.properties b/search-server/build.properties deleted file mode 100644 index b8fd474..0000000 --- a/search-server/build.properties +++ /dev/null @@ -1,22 +0,0 @@ -java.src.dir=src/java -config.src.dir=src/config -java.lib.dir=lib -java.task.lib.dir=task-lib -report.dir=report -build.dir=build -build.instrument.dir=instrumented -dist.dir=dist -jar.name=spacewalk-search -jar.version=1.0 -jar.file.name=${jar.name}-${jar.version}.jar -ivyserver= http://jmrodri.fedorapeople.org -ivy.log.module.when.found=false -ivy.log.resolved.revision=false -# Checkstyle config entries -checkstyle.cache.file=.checkstyle_cache -checkstyle.header.file=./buildconf/LICENSE.txt -javadoc.method.scope=public -javadoc.type.scope=package -javadoc.var.scope=package -javadoc.lazy=true -jpackage.jars="commons-lang commons-logging redstone-xmlrpc redstone-xmlrpc-client quartz tanukiwrapper log4j commons-httpclient oro commons-cli commons-codec" diff --git a/search-server/build.xml b/search-server/build.xml deleted file mode 100644 index de8e25f..0000000 --- a/search-server/build.xml +++ /dev/null @@ -1,241 +0,0 @@ -<project name="search-server" basedir="." default="all"> - <import file="buildconf/build-utils.xml" /> - - <property file="build.properties" /> - - <path id="project.taskjars"> - <fileset dir="${java.task.lib.dir}" includes="**/*.jar" /> - </path> - - <path id="project.classpath"> - <fileset dir="${java.lib.dir}" includes="**/*.jar" /> - <fileset dir="${java.task.lib.dir}" > - <include name="junit*.jar" /> - <include name="emma*.jar" /> - </fileset> - </path> - - <path id="test.classpath"> - <pathelement location="${build.dir}" /> - <path refid="project.classpath" /> - <pathelement location="${basedir}/src/config" /> - </path> - - <target name="init-tasks"> - <taskdef name="ivy-retrieve" - classname="fr.jayasoft.ivy.ant.IvyRetrieve" - classpathref="project.taskjars"/> - <taskdef name="ivy-configure" - classname="fr.jayasoft.ivy.ant.IvyConfigure" - classpathref="project.taskjars"/> - <taskdef name="ivy-resolve" - classname="fr.jayasoft.ivy.ant.IvyResolve" - classpathref="project.taskjars"/> - <taskdef name="ivy-publish" - classname="fr.jayasoft.ivy.ant.IvyPublish" - classpathref="project.taskjars"/> - <taskdef resource="checkstyletask.properties"> - <classpath> - <pathelement path="${java.task.lib.dir}/checkstyle-4.1.jar" /> - <pathelement path="${java.task.lib.dir}/commons-beanutils-1.7.0.jar" /> - <pathelement path="${java.task.lib.dir}/commons-logging-1.0.4.jar" /> - <pathelement path="${java.task.lib.dir}/antlr-2.7.6.jar" /> - <pathelement path="${java.task.lib.dir}/regexp-1.3.jar" /> - </classpath> - </taskdef> - <taskdef resource="emma_ant.properties" classpathref="project.taskjars" /> - <taskdef resource="net/sf/antcontrib/antlib.xml"> - <classpath> - <pathelement location="${java.task.lib.dir}/ant-contrib-1.0.jar"/> - </classpath> - </taskdef> - </target> - - <target name="check-testcase-name" if="testcase" depends="init-tasks"> - <propertyregex property="valid.testcase" - input="${testcase}" - regexp="(.*)Test$" - select="\1" - casesensitive="true" /> - </target> - - <target name="check-testcase" depends="check-testcase-name" if="testcase" - unless="valid.testcase"> - <fail message="Invalid testcase name: ${testcase}. Class name must look like .*Test"/> - </target> - - <target name="test" depends="compile, check-testcase" - description="Run the tests (use -Dtestcase=<testcase>)"> - <mkdir dir="${report.dir}" /> - <!-- set testcase to wildcard "*" to test everything, unless - a specific testcase name was supplied. --> - <if> - <not> - <isset property="${testcase}" /> - </not> - <then> - <property name="testcase" value="*Test"/> - </then> - </if> - <junit> - <sysproperty key="rhn.config.dir" value="/etc/rhn" /> - <sysproperty key="search.config.dir" value="src/config/search" /> - <sysproperty key="log4j.configuration" value="log4j.test.properties" /> - <classpath refid="test.classpath" /> - <formatter type="brief" usefile="false" /> - <formatter type="xml" /> - <batchtest fork="yes" todir="${report.dir}"> - <fileset dir="${build.dir}" includes="**/${testcase}.class" /> - </batchtest> - </junit> - </target> - - <target name="checkstyle" depends="init-tasks, package"> - <delete dir="${report.dir}" /> - <mkdir dir="${report.dir}" /> - <checkstyle config="checkstyle.xml" failOnViolation="true" > - <property key="javadoc.method.scope" value="public" /> - <property key="javadoc.type.scope" value="package" /> - <property key="javadoc.var.scope" value="package" /> - <property key="javadoc.lazy" value="${javadoc.lazy}" /> - <fileset dir="${java.src.dir}" > - <include name="**/*.java" /> - <exclude name="**/tests/**" /> - </fileset> - <classpath> - <fileset dir="${java.lib.dir}" includes="**/*.jar" /> - <fileset dir="${java.task.lib.dir}" includes="junit*.jar" /> - <fileset dir="${dist.dir}" includes="${jar.file.name}" /> - </classpath> - <formatter type="plain" /> - <formatter type="xml" toFile="${report.dir}/checkstyle_report.xml" /> - </checkstyle> - </target> - - <target name="resolve-local" description="resolve jars via jpackage" if="installbuild"> - <copy toDir="${java.lib.dir}" overwrite="true"> - <fileset dir="buildconf/tempjars"> - <include name="**/*.jar" /> - </fileset> - </copy> - - <jpackage-deps jars="${jpackage.jars}" dir="${java.lib.dir}" /> - </target> - - <target name="resolve-ivy" description="retrieve dependencies with ivy" - depends="init-tasks" unless="installbuild"> - <!-- properties set here to show we are overriding default ivy values --> - <property name="ivy.default.ivy.user.dir" value="${user.home}/.ivy/search-server"/> - <property name="ivy.local.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> - <property name="ivy.ivyrep.default.ivy.root" value="${ivyserver}/ivy/" /> - <property name="ivy.ivyrep.default.artifact.root" value="${ivyserver}/ivy/" /> - <property name="ivy.ivyrep.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> - <ivy-retrieve/> - </target> - - <target name="resolve" depends="resolve-ivy,resolve-local" /> - - <target name="prep"> - <mkdir dir="${build.dir}" /> - <mkdir dir="${dist.dir}" /> - <mkdir dir="${java.lib.dir}" /> - </target> - - <target name="clean"> - <delete dir="${build.dir}" /> - <delete dir="${build.instrument.dir}" /> - <delete dir="${dist.dir}" /> - <delete dir="${java.lib.dir}" /> - <delete dir="${report.dir}" /> - <delete file="coverage.ec" /> - <delete file="rhn_search_daemon.log" /> - </target> - - <target name="compile" depends="prep, resolve"> - <javac srcdir="${java.src.dir}" - classpathref="project.classpath" - destdir="${build.dir}" - deprecation="on" - excludesfile="buildconf/exclude" - debug="on" /> - <copy todir="${build.dir}/com/redhat/satellite/search/db"> - <fileset dir="${config.src.dir}/com/redhat/satellite/search/db/"> - <include name="*.xml" /> - </fileset> - </copy> - <copy file="${config.src.dir}/log4j.properties" todir="${build.dir}" /> - </target> - - <target name="package" depends="compile"> - <jar basedir="${build.dir}" - destfile="${dist.dir}/${jar.file.name}" - excludes="**/tests/**" - includes="**/**" /> - </target> - - <target name="create-local-repo" description="Create a local Ivy repo"> - <mkdir dir="${user.home}/.ivy/search-server/local/" /> - </target> - - <target name="clean-cache" description="Removes the Ivy jar cache"> - <delete dir="${lib.dir}" quiet="true" /> - <delete dir="${user.home}/.ivy/search-server/cache/" quiet="true" /> - </target> - - <target name="all" depends="package" /> - - <target name="init-install"> - <property name="installbuild" value="true"/> - </target> - - <target name="install" depends="init-install,package" /> - - <target name="instrument" depends="init-tasks"> - <mkdir dir="${report.dir}" /> - <mkdir dir="${build.instrument.dir}" /> - - <emma enabled="true"> - <instr merge="no" - mode="overwrite" - instrpath="${build.dir}" - outfile="${report.dir}/coverage.em" > - <filter excludes="*Exception" /> - <filter excludes="*.tests.*" /> - </instr> - <!-- - <instr outdir="${build.instrument.dir}" merge="no" - instrpath="${build.dir}" - mode="fullcopy" - outfile="${report.dir}/coverage.em" > - <filter excludes="*Exception" /> - <filter excludes="*.tests.*" /> - </instr> - --> - </emma> - - <!-- - <copy toDir="${build.instrument.dir}/classes"> - <fileset dir="${build.dir}"> - <include name="**/*.xml" /> - </fileset> - </copy> - --> - </target> - - <target name="coverage-report" depends="instrument"> - <emma enabled="true"> - <report sourcepath="${java.src.dir}" - sort="+block,+name,+method,+class" - metrics="method:70,block:80,line:80,class:100"> - <fileset dir="${report.dir}" includes="*.em" /> - <fileset dir="." includes="*.ec" /> - <html outfile="${report.dir}/emma.html" - depth="method" - columns="name,class,method,block,line" /> - <xml outfile="${report.dir}/emma.xml" - depth="method" - columns="name,class,method,block,line" /> - </report> - </emma> - </target> -</project> diff --git a/search-server/buildconf/LICENSE.txt b/search-server/buildconf/LICENSE.txt deleted file mode 100644 index 629462a..0000000 --- a/search-server/buildconf/LICENSE.txt +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ diff --git a/search-server/buildconf/build-utils.xml b/search-server/buildconf/build-utils.xml deleted file mode 100644 index 0011a83..0000000 --- a/search-server/buildconf/build-utils.xml +++ /dev/null @@ -1,19 +0,0 @@ -<project name="build-utils.xml"> - <!-- - creates symlinks for jpackage jars - requires: jpackage-utils - other: requires rpms of the jars specified to be installed - --> - <macrodef name="jpackage-deps"> - <attribute name="jars" /> - <attribute name="dir" /> - - <sequential> - <exec executable="build-jar-repository" failonerror="true" > - <arg value="--preserve-naming" /> - <arg value="-s @{dir}" /> - <arg line="@{jars}" /> - </exec> - </sequential> - </macrodef> -</project> diff --git a/search-server/buildconf/exclude b/search-server/buildconf/exclude deleted file mode 100644 index 7873251..0000000 --- a/search-server/buildconf/exclude +++ /dev/null @@ -1,2 +0,0 @@ -**/WebCrawl.java -**/IndexDocumentsTask.java diff --git a/search-server/buildconf/tempjars/hadoop-0.18.1-core.jar b/search-server/buildconf/tempjars/hadoop-0.18.1-core.jar deleted file mode 100644 index a024c3c..0000000 Binary files a/search-server/buildconf/tempjars/hadoop-0.18.1-core.jar and /dev/null differ diff --git a/search-server/buildconf/tempjars/ibatis-2.3.0.677.jar b/search-server/buildconf/tempjars/ibatis-2.3.0.677.jar deleted file mode 100644 index 72dbe19..0000000 Binary files a/search-server/buildconf/tempjars/ibatis-2.3.0.677.jar and /dev/null differ diff --git a/search-server/buildconf/tempjars/lucene-analyzers-2.3.2.jar b/search-server/buildconf/tempjars/lucene-analyzers-2.3.2.jar deleted file mode 100644 index cf0d296..0000000 Binary files a/search-server/buildconf/tempjars/lucene-analyzers-2.3.2.jar and /dev/null differ diff --git a/search-server/buildconf/tempjars/lucene-core-2.3.2.jar b/search-server/buildconf/tempjars/lucene-core-2.3.2.jar deleted file mode 100644 index 639f750..0000000 Binary files a/search-server/buildconf/tempjars/lucene-core-2.3.2.jar and /dev/null differ diff --git a/search-server/buildconf/tempjars/nutch-2008-12-01_04-01-21.jar b/search-server/buildconf/tempjars/nutch-2008-12-01_04-01-21.jar deleted file mode 100644 index 4bdec09..0000000 Binary files a/search-server/buildconf/tempjars/nutch-2008-12-01_04-01-21.jar and /dev/null differ diff --git a/search-server/buildconf/tempjars/picocontainer-1.3.jar b/search-server/buildconf/tempjars/picocontainer-1.3.jar deleted file mode 100644 index 9a3f670..0000000 Binary files a/search-server/buildconf/tempjars/picocontainer-1.3.jar and /dev/null differ diff --git a/search-server/buildconf/tempjars/simple-core-3.1.3.jar b/search-server/buildconf/tempjars/simple-core-3.1.3.jar deleted file mode 100644 index 80b44c1..0000000 Binary files a/search-server/buildconf/tempjars/simple-core-3.1.3.jar and /dev/null differ diff --git a/search-server/checkstyle.xml b/search-server/checkstyle.xml deleted file mode 100644 index 89cde80..0000000 --- a/search-server/checkstyle.xml +++ /dev/null @@ -1,233 +0,0 @@ -<?xml version="1.0"?> -<!DOCTYPE module PUBLIC - "-//Puppy Crawl//DTD Check Configuration 1.1//EN" - "http://www.puppycrawl.com/dtds/configuration_1_1.dtd"> - -<!-- - - Checkstyle configuration that checks the sun coding conventions from: - - - the Java Language Specification at - http://java.sun.com/docs/books/jls/second_edition/html/index.html - - - the Sun Code Conventions at http://java.sun.com/docs/codeconv/ - - - the Javadoc guidelines at - http://java.sun.com/j2se/javadoc/writingdoccomments/index.html - - - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html - - - some best practices - - Checkstyle is very configurable. Be sure to read the documentation at - http://checkstyle.sf.net (or in your downloaded distribution). - - Most Checks are configurable, be sure to consult the documentation. - - To completely disable a check, just comment it out or delete it from the file. - - Finally, it is worth reading the documentation. - ---> - -<module name="Checker"> - - <!-- Checks that a package.html file exists for each package. --> - <!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml --> -<!-- <module name="PackageHtml"/> --> - - <!-- Checks whether files end with a new line. --> - <!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile --> - <module name="NewlineAtEndOfFile"/> - - <!-- Checks that property files contain the same keys. --> - <!-- See http://checkstyle.sf.net/config_misc.html#Translation --> - <module name="Translation"/> - - <module name="TreeWalker"> - - <property name="cacheFile" value="${checkstyle.cache.file}"/> - - <!-- ************************************************************** --> - <!-- Checks that are different from the sun coding conventions ones --> - <!-- ************************************************************** --> - <property name="tabWidth" value="4"/> - <module name="LeftCurly" /> - <module name="RightCurly"> - <property name="option" value="alone"/> - </module> - <module name="LineLength"> - <property name="ignorePattern" value="import"/> - <property name="max" value="92"/> - </module> - - <!-- ************************************************************** --> - <!-- Default Sun coding conventions checks --> - <!-- ************************************************************** --> - - <!-- Checks for Javadoc comments. --> - <!-- See http://checkstyle.sf.net/config_javadoc.html --> - <module name="JavadocMethod"> - <property name="scope" value="${javadoc.method.scope}" /> - <property name="allowUndeclaredRTE" value="true" /> - <property name="allowThrowsTagsForSubclasses" value="${javadoc.lazy}" /> - <property name="allowMissingParamTags" value="${javadoc.lazy}" /> - <property name="allowMissingThrowsTags" value="${javadoc.lazy}" /> - <property name="allowMissingReturnTag" value="${javadoc.lazy}" /> - </module> - <module name="JavadocType"> - <property name="versionFormat" value="\$Rev.*\$"/> - <property name="scope" value="${javadoc.type.scope}" /> - </module> - <module name="JavadocVariable"> - <property name="scope" value="${javadoc.var.scope}" /> - <property name="severity" value="ignore"/> - </module> - -<!-- JavadocStyle not supported by some versions. --> -<!-- <module name="JavadocStyle"/> --> - - - <!-- Checks for Naming Conventions. --> - <!-- See http://checkstyle.sf.net/config_naming.html --> - <module name="ConstantName"/> - <module name="LocalFinalVariableName"/> - <module name="LocalVariableName"/> - <module name="MethodName"/> - <module name="PackageName"> - <property name="format" value="^[a-z]+(\.[A-Za-z][A-Za-z0-9]*)*$"/> - </module> - <module name="ParameterName"/> - <module name="StaticVariableName"/> - <module name="TypeName"/> - - - <!-- Checks for Headers --> - <!-- See http://checkstyle.sf.net/config_header.html --> - <module name="Header"> - <!-- The follow property value demonstrates the ability --> - <!-- to have access to ANT properties. In this case it uses --> - <!-- the ${basedir} property to allow Checkstyle to be run --> - <!-- from any directory within a project. --> - <property name="headerFile" value="${checkstyle.header.file}"/> - </module> - - <!-- Following interprets the header file as regular expressions. --> - <!-- <module name="RegexpHeader"/> --> - - - - <!-- Checks for imports --> - <!-- See http://checkstyle.sf.net/config_import.html --> - <module name="AvoidStarImport"/> - <module name="IllegalImport"/> <!-- defaults to sun.* packages --> - <module name="RedundantImport"/> - <module name="UnusedImports"/> - - - <!-- Checks for Size Violations. --> - <!-- See http://checkstyle.sf.net/config_sizes.html --> - <module name="MethodLength"> - <property name="tokens" value="METHOD_DEF"/> - <property name="max" value="150"/> - <property name="countEmpty" value="false"/> - </module> - <module name="ParameterNumber"/> - - - <!-- Checks for whitespace --> - <!-- See http://checkstyle.sf.net/config_whitespace.html --> - <module name="EmptyForIteratorPad"/> - <module name="NoWhitespaceAfter"> - <property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, UNARY_PLUS, TYPECAST, GENERIC_START"/> - </module> - <module name="NoWhitespaceBefore"> - <property name="tokens" value="SEMI, POST_DEC, POST_INC, GENERIC_START, GENERIC_END"/> - </module> - <module name="OperatorWrap"> - <property name="option" value="eol" /> - </module> - <module name="ParenPad"/> -<!-- <module name="TypecastParenPad"/> --> - <module name="TabCharacter"/> - <module name="WhitespaceAfter"> - <property name="tokens" value="COMMA, SEMI, GENERIC_END"/> - </module> - <module name="WhitespaceAround"> - <property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, -COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LCURLY, LE, LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, -LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS, -MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION, RCURLY, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN, STAR, STAR_ASSIGN, TYPE_EXTENSION_AND, WILDCARD_TYPE"/> - </module> - <module name="MethodParamPad"> - <property name="allowLineBreaks" value="true"/> - </module> - - - <!-- Modifier Checks --> - <!-- See http://checkstyle.sf.net/config_modifiers.html --> - <module name="ModifierOrder"/> - <module name="RedundantModifier"/> - - - <!-- Checks for blocks. You know, those {}'s --> - <!-- See http://checkstyle.sf.net/config_blocks.html --> - <module name="AvoidNestedBlocks"/> - <module name="EmptyBlock"> - <property name="option" value="text"/> - </module> - - <module name="NeedBraces"/> - - - <!-- Checks for common coding problems --> - <!-- See http://checkstyle.sf.net/config_coding.html --> - <!-- <module name="AvoidInlineConditionals"/> --> -<!-- <module name="CovariantEquals"/> --> - <module name="DoubleCheckedLocking"/> <!-- MY FAVOURITE --> - <module name="EmptyStatement"/> - <module name="EqualsHashCode"/> - <module name="HiddenField"/> - <module name="IllegalInstantiation"/> - <module name="InnerAssignment"/> - <!-- <module name="MagicNumber" /> --> - <module name="MissingSwitchDefault"/> -<!-- <module name="RedundantThrows"/> --> - <module name="SimplifyBooleanExpression"/> - <module name="SimplifyBooleanReturn"/> -<!-- <module name="IllegalCatch"/> --> -<!-- <module name="PackageDeclaration"/> --> - - <!-- Checks for class design --> - <!-- See http://checkstyle.sf.net/config_design.html --> -<!-- <module name="DesignForExtension"/>--> -<!-- <module name="FinalClass"/> --> - <module name="HideUtilityClassConstructor"/> - <module name="InterfaceIsType"/> - <module name="VisibilityModifier"> - <property name="protectedAllowed" value="true" /> - </module> - - - <!-- Miscellaneous other checks. --> - <!-- See http://checkstyle.sf.net/config_misc.html --> - <module name="ArrayTypeStyle"/> -<!-- <module name="FinalParameters"/> --> -<!-- - <module name="GenericIllegalRegexp"> - <property name="format" value="System\.out\.println"/> - </module> ---> - <module name="UpperEll"/> - - <!-- Optional Checks --> - <!-- (gack, not available until version 3 of checkstyle --> -<!-- - <module name="usage.OneMethodPrivateField"/> - <module name="usage.UnusedLocalVariable"/> - <module name="usage.UnusedParameter"/> - <module name="usage.UnusedPrivateField"/> ---> - </module> - -</module> diff --git a/search-server/ivy.xml b/search-server/ivy.xml deleted file mode 100644 index aec44ca..0000000 --- a/search-server/ivy.xml +++ /dev/null @@ -1,28 +0,0 @@ -<ivy-module version="1.0"> - <info organisation="redhat" module="rhn-java" /> - <dependencies> - <dependency org="redhat" name="simple-core" rev="3.1.3" /> - <dependency org="redhat" name="lucene-core" rev="2.3.2" /> - <dependency org="redhat" name="lucene-analyzers" rev="2.3.2" /> - <dependency org="redhat" name="commons-lang" rev="2.0" /> - <dependency org="redhat" name="commons-logging" rev="1.1" /> - <dependency org="redhat" name="redstone-xmlrpc" rev="1.1_20071120" /> - <dependency org="redhat" name="redstone-xmlrpc-client" rev="1.1_20071120" /> - <dependency org="redhat" name="ibatis" rev="2.3.0.677" /> - <dependency org="redhat" name="ojdbc14" rev="1.10" /> - <dependency org="redhat" name="quartz" rev="1.5.1" /> - <dependency org="redhat" name="picocontainer" rev="1.3" /> - <dependency org="redhat" name="tanukiwrapper" rev="3.1.2" /> - <dependency org="redhat" name="log4j" rev="1.2.8" /> - <dependency org="redhat" name="logdriver" rev="0.1" /> - <dependency org="redhat" name="nutch" rev="2008-12-01_04-01-21" /> - <dependency org="redhat" name="hadoop" rev="0.18.1-core" /> - <!-- dependency org="redhat" name="commons-httpclient" rev="3.0" / --> - <!-- dependency org="redhat" name="oro" rev="2.0.8" / --> - <!-- dependency org="redhat" name="commons-cli" rev="1.0" / --> - <!-- dependency org="redhat" name="commons-codec" rev="1.2" / --> - <!-- dependency org="redhat" name="tika" rev="0.1-incubating" / --> - <!-- dependency org="redhat" name="icu4j" rev="3_8_1" / --> - <!-- dependency org="redhat" name="icu4j-charsets" rev="3_8_1" / --> - </dependencies> -</ivy-module> diff --git a/search-server/libsrc/hadoop-0.16.4-src.jar b/search-server/libsrc/hadoop-0.16.4-src.jar deleted file mode 100644 index c941a7c..0000000 Binary files a/search-server/libsrc/hadoop-0.16.4-src.jar and /dev/null differ diff --git a/search-server/libsrc/ibatis-2.3.0.677-src.zip b/search-server/libsrc/ibatis-2.3.0.677-src.zip deleted file mode 100644 index 8c3274b..0000000 Binary files a/search-server/libsrc/ibatis-2.3.0.677-src.zip and /dev/null differ diff --git a/search-server/libsrc/icu4j-3_8_1-src.jar b/search-server/libsrc/icu4j-3_8_1-src.jar deleted file mode 100644 index 19dd665..0000000 Binary files a/search-server/libsrc/icu4j-3_8_1-src.jar and /dev/null differ diff --git a/search-server/libsrc/log4j-1.2.8-src.jar b/search-server/libsrc/log4j-1.2.8-src.jar deleted file mode 100644 index b36df55..0000000 Binary files a/search-server/libsrc/log4j-1.2.8-src.jar and /dev/null differ diff --git a/search-server/libsrc/lucene-2.3.2-src.zip b/search-server/libsrc/lucene-2.3.2-src.zip deleted file mode 100644 index b49b9b6..0000000 Binary files a/search-server/libsrc/lucene-2.3.2-src.zip and /dev/null differ diff --git a/search-server/libsrc/nutch-1.0dev_2008_05_14-src.jar b/search-server/libsrc/nutch-1.0dev_2008_05_14-src.jar deleted file mode 100644 index 372eca5..0000000 Binary files a/search-server/libsrc/nutch-1.0dev_2008_05_14-src.jar and /dev/null differ diff --git a/search-server/libsrc/nutch-2008-12-01_04-01-21-src.zip b/search-server/libsrc/nutch-2008-12-01_04-01-21-src.zip deleted file mode 100644 index 268a010..0000000 Binary files a/search-server/libsrc/nutch-2008-12-01_04-01-21-src.zip and /dev/null differ diff --git a/search-server/libsrc/quartz-1.5.1-src.jar b/search-server/libsrc/quartz-1.5.1-src.jar deleted file mode 100644 index 79edf1a..0000000 Binary files a/search-server/libsrc/quartz-1.5.1-src.jar and /dev/null differ diff --git a/search-server/libsrc/redstone-xmlrpc-1.1_20071120-src.jar b/search-server/libsrc/redstone-xmlrpc-1.1_20071120-src.jar deleted file mode 100644 index 2dc7cf7..0000000 Binary files a/search-server/libsrc/redstone-xmlrpc-1.1_20071120-src.jar and /dev/null differ diff --git a/search-server/libsrc/simple-3.1.3-src.zip b/search-server/libsrc/simple-3.1.3-src.zip deleted file mode 100644 index cdd83c6..0000000 Binary files a/search-server/libsrc/simple-3.1.3-src.zip and /dev/null differ diff --git a/search-server/libsrc/tanukiwrapper-3.1.2-src.jar b/search-server/libsrc/tanukiwrapper-3.1.2-src.jar deleted file mode 100644 index 9436fbd..0000000 Binary files a/search-server/libsrc/tanukiwrapper-3.1.2-src.jar and /dev/null differ diff --git a/search-server/nutch/crawl_jsp.sh b/search-server/nutch/crawl_jsp.sh deleted file mode 100755 index d7a7e19..0000000 --- a/search-server/nutch/crawl_jsp.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - - -if [ "${JAVA_HOME}" = "" ]; then - export JAVA_HOME=/usr/lib/jvm/java - echo "Setting JAVA_HOME to: ${JAVA_HOME}" -fi - -export NUTCH_HOME=/usr/share/nutch -export NUTCH_CONF_DIR=`pwd`/crawl_jsp/conf -export NUTCH_OPTS=-Djava.util.logging.config.file="${NUTCH_CONF}/logging.properties" -export NUTCH_LOG_DIR=`pwd`/logs -export NUTCH_LOG_FILE=$0.log -export OUTPUT_DIR=`pwd`/data/crawl_jsp - -echo "NUTCH_HOME = ${NUTCH_HOME}" -echo "NUTCH_CONF_DIR = ${NUTCH_CONF_DIR}" -echo "NUTCH_OPTS = ${NUTCH_OPTS}" -echo "NUTCH_LOG_DIR = ${NUTCH_LOG_DIR}" -echo "NUTCH_LOG_FILE = ${NUTCH_LOG_FILE}" -echo "OUTPUT_DIR = ${OUTPUT_DIR}" - -if [ ! -d ${NUTCH_LOG_DIR} ]; then - echo "Creating log directory ${NUTCH_LOG_DIR}" - mkdir ${NUTCH_LOG_DIR} -fi - - -if [ ! -d ${OUTPUT_DIR} ]; then - echo "Creating output directory ${OUTPUT_DIR}" - mkdir -p ${OUTPUT_DIR} -fi -#Need to adjust nutch RPM so it's scripts under bin are executable by all - -${NUTCH_HOME}/bin/nutch crawl ${NUTCH_CONF_DIR}/../urls -dir ${OUTPUT_DIR} -depth 10 -threads 50 diff --git a/search-server/nutch/crawl_jsp/conf/automaton-urlfilter.txt b/search-server/nutch/crawl_jsp/conf/automaton-urlfilter.txt deleted file mode 100644 index 84cf8fa..0000000 --- a/search-server/nutch/crawl_jsp/conf/automaton-urlfilter.txt +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The default url filter. -# Better for whole-internet crawling. - -# Each non-comment, non-blank line contains a regular expression -# prefixed by '+' or '-'. The first matching pattern in the file -# determines whether a URL is included or ignored. If no pattern -# matches, the URL is ignored. - -# skip file: ftp: and mailto: urls --(file|ftp|mailto):.* - -# skip image and other suffixes we can't yet parse --.*.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe) - -# skip URLs containing certain characters as probable queries, etc. --.*[?*!@=].* - -# accept anything else -+.* diff --git a/search-server/nutch/crawl_jsp/conf/common-terms.utf8 b/search-server/nutch/crawl_jsp/conf/common-terms.utf8 deleted file mode 100644 index 220d1da..0000000 --- a/search-server/nutch/crawl_jsp/conf/common-terms.utf8 +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Common terms and phrases which will be indexed in n-grams -# in order to optimize search. -content:a -content:and -content:for -content:in -content:of -content:the -content:to -url:com -url:http -url:http-www -url:www diff --git a/search-server/nutch/crawl_jsp/conf/configuration.xsl b/search-server/nutch/crawl_jsp/conf/configuration.xsl deleted file mode 100644 index 79141dc..0000000 --- a/search-server/nutch/crawl_jsp/conf/configuration.xsl +++ /dev/null @@ -1,40 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> -<xsl:output method="html"/> -<xsl:template match="configuration"> -<html> -<body> -<table border="1"> -<tr> - <td>name</td> - <td>value</td> - <td>description</td> -</tr> -<xsl:for-each select="property"> -<tr> - <td><a name="{name}"><xsl:value-of select="name"/></a></td> - <td><xsl:value-of select="value"/></td> - <td><xsl:value-of select="description"/></td> -</tr> -</xsl:for-each> -</table> -</body> -</html> -</xsl:template> -</xsl:stylesheet> diff --git a/search-server/nutch/crawl_jsp/conf/context.xsl b/search-server/nutch/crawl_jsp/conf/context.xsl deleted file mode 100644 index fbb3bc6..0000000 --- a/search-server/nutch/crawl_jsp/conf/context.xsl +++ /dev/null @@ -1,88 +0,0 @@ -<?xml version="1.0" encoding="ISO-8859-1"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Author : Chris Mattmann - Auhtor : Jérôme Charron - Description: This xsl file is used to transform dynamic properties out of the - nutch-default.xml file into a deployable Context.xml file for configuring - the Nutch war file in a servlet container. ---> - -<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform%22%3E - -<xsl:template match="/"> -<Context path="/nutch" docBase="nutch-0.9-dev.war" - debug="5" reloadable="true" crossContext="true"> - -<xsl:for-each select="configuration/property"> - -<!-- "searcher." properties --> -<xsl:call-template name="parameter"> - <xsl:with-param name="property" select="current()"/> - <xsl:with-param name="filter" select="'searcher.'"/> -</xsl:call-template> - -<!-- "plugin." properties --> -<xsl:call-template name="parameter"> - <xsl:with-param name="property" select="current()"/> - <xsl:with-param name="filter" select="'plugin.'"/> -</xsl:call-template> - -<!-- "extension.clustering." properties --> -<xsl:call-template name="parameter"> - <xsl:with-param name="property" select="current()"/> - <xsl:with-param name="filter" select="'extension.clustering.'"/> -</xsl:call-template> - -<!-- "extension.ontology." properties --> -<xsl:call-template name="parameter"> - <xsl:with-param name="property" select="current()"/> - <xsl:with-param name="filter" select="'extension.ontology.'"/> -</xsl:call-template> - -<!-- "query." properties --> -<xsl:call-template name="parameter"> - <xsl:with-param name="property" select="current()"/> - <xsl:with-param name="filter" select="'query.'"/> -</xsl:call-template> - -</xsl:for-each> - -</Context> -</xsl:template> - - -<!-- - ! Template used to write out a parameter if the property's - ! name contains the specified filter string. - !--> -<xsl:template name="parameter"> - <xsl:param name="property"/> - <xsl:param name="filter"/> - <xsl:if test="contains(name, $filter)"> - <Parameter override="false"> - <xsl:attribute name="name"> - <xsl:value-of select="name"/> - </xsl:attribute> - <xsl:attribute name="value"> - <xsl:value-of select="value"/> - </xsl:attribute> - </Parameter> - </xsl:if> -</xsl:template> - -</xsl:stylesheet> diff --git a/search-server/nutch/crawl_jsp/conf/crawl-tool.xml b/search-server/nutch/crawl_jsp/conf/crawl-tool.xml deleted file mode 100644 index 9f6d667..0000000 --- a/search-server/nutch/crawl_jsp/conf/crawl-tool.xml +++ /dev/null @@ -1,59 +0,0 @@ -<?xml version="1.0" ?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<!-- Overidden defaults for intranet use. --> - -<!-- Do not modify this file directly. Instead, copy entries that you --> -<!-- wish to modify from this file into nutch-site.xml and change them --> -<!-- there. If nutch-site.xml does not already exist, create it. --> - -<configuration> - -<property> - <name>urlfilter.regex.file</name> - <value>crawl-urlfilter.txt</value> -</property> - -<property> - <name>db.ignore.internal.links</name> - <value>false</value> - <description>If true, when adding new links to a page, links from - the same host are ignored. This is an effective way to limit the - size of the link database, keeping the only the highest quality - links. - </description> -</property> - -<property> - <name>fetcher.server.delay</name> - <value>1.0</value> - <description>The number of seconds the fetcher will delay between - successive requests to the same server.</description> -</property> - -<property> - <name>http.max.delays</name> - <value>1000</value> - <description>The number of times a thread will delay when trying to - fetch a page. When using the crawl tool there are likely to be very - few different hosts, so we need to be willing to wait longer for - each.</description> -</property> - -</configuration> diff --git a/search-server/nutch/crawl_jsp/conf/crawl-urlfilter.txt b/search-server/nutch/crawl_jsp/conf/crawl-urlfilter.txt deleted file mode 100644 index 2698c17..0000000 --- a/search-server/nutch/crawl_jsp/conf/crawl-urlfilter.txt +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# The url filter file used by the crawl command. - -# Better for intranet crawling. -# Be sure to change MY.DOMAIN.NAME to your domain name. - -# Each non-comment, non-blank line contains a regular expression -# prefixed by '+' or '-'. The first matching pattern in the file -# determines whether a URL is included or ignored. If no pattern -# matches, the URL is ignored. - -# skip file:, ftp:, & mailto: urls -#-^(file|ftp|mailto): --^(http|ftp|mailto): - -# skip image and other suffixes we can't yet parse --.(gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|jpeg|JPEG|bmp|BMP|pdf)$ - -# skip URLs containing certain characters as probable queries, etc. --[?*!@=] - -# skip URLs with slash-delimited segment that repeats 3+ times, to break loops --.*(/[^/]+)/[^/]+\1/[^/]+\1/ - -# accept hosts in MY.DOMAIN.NAME -#+^http://(%5Ba-z0-9%5D*%5C.)*MY.DOMAIN.NAME/ -#+^http://(%5Ba-z0-9%5D*%5C.)*redhat.com/ - -# skip everything else -#- - - -#accept everything else -#+.* - -#ingore all svn entries --.*.svn -#only read in english docs from these locations -+.*/docs/guides/channel-mgmt/en/ -+.*/docs/guides/client-config/en/ -+.*/docs/guides/proxy/rhn511/en/ -+.*/docs/release-notes/satellite/rhn510/en-US/ -+.*/docs/guides/satellite/en/ -+.*/docs/guides/reference/en/ -#skip everything else --.* diff --git a/search-server/nutch/crawl_jsp/conf/domain-suffixes.xml b/search-server/nutch/crawl_jsp/conf/domain-suffixes.xml deleted file mode 100644 index 635f5a2..0000000 --- a/search-server/nutch/crawl_jsp/conf/domain-suffixes.xml +++ /dev/null @@ -1,4354 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- - Document : domain-suffixes.xml - Author : Enis Soztutar - enis.soz.nutch@gmail.com - Description: This document contains top level domains - as described by the Internet Assigned Numbers - Authotiry (IANA), and second or third level domains that - are known to be managed by domain registerers. People at - Mozilla community call these public suffixes or effective - tlds. There is no algorithmic way of knowing whether a suffix - is a public domain suffix, or not. So this large file is used - for this purpose. The entries in the file is used to find the - domain of a url, which may not the same thing as the host of - the url. For example for "http://lucene.apache.org/nutch" the - hostname is lucene.apache.org, however the domain name for this - url would be apache.org. Domain names can be quite handy for - statistical analysis, and fighting against spam. - - The list of TLDs is constructed from IANA, and the - list of "effective tlds" are constructed from Wikipedia, - http://wiki.mozilla.org/TLD_List, and http://publicsuffix.org/ - The list may not include all the suffixes, but some - effort has been spent to make it comprehensive. Please forward - any improvements for this list to nutch-dev mailing list, or - nutch JIRA. - - Top level domains(tlds) are grouped - to three, namely infrastrusture, generic and country - code tlds. Infrastrusture tlds are only used for - technical reasons. Generic tlds represents the type - of the organization that they represent. Those in - current use and those waiting for approval is listed. - Most of the country code tlds correspond to the two - letter ISO-3166-1 country codes. - Each tld is listed with its domain (such as com), a - status enumeration describing the status of the tld, - and optionally a description or description for convenience. - cctlds are listed with additional country name field. - - status and boost elements are optional, with default values IN_USE - and 1.0 respectively. see domain-suffixes.xsd for the xml schema. - - - References : - http://www.iana.org - http://www.iana.org/gtld/gtld.htm - http://www.iana.org/root-whois/index.html - http://en.wikipedia.org/wiki/Top-level_domain - http://en.wikipedia.org/wiki/List_of_Internet_top-level_domains - http://wiki.mozilla.org/TLD_List - http://publicsuffix.org/ - https://bugzilla.mozilla.org/show_bug.cgi?id=331510 - http://www.neuhaus.com/domaincheck/domain_list.htm ---> - -<domains xmlns="http://lucene.apache.org/nutch" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://lucene.apache.org/nutch domain-suffixes.xsd"> - - <tlds> - <!-- Infrastructure Top Level Domains --> - <itlds> - <tld domain="root"> - <status>INFRASTRUCTURE</status> - <description> - (from http://en.wikipedia.org/wiki/.root) - vrsn-end-of-zone-marker-dummy-record.root is a domain name - listed in the DNS root zone as a diagnostic marker, whose - presence demonstrates the root zone was not truncated upon - loading by a root nameserver. It could be argued it represents - a top-level domain of .root, although technically no such - delegation exists. - </description> - </tld> - - <tld domain="arpa"> - <status>INFRASTRUCTURE</status> - <description> - (from http://en.wikipedia.org/wiki/.arpa) .arpa is an Internet - top-level domain (TLD) used exclusively for - Internet-infrastructure purposes. It does not function as a - normal TLD where websites are registered, but rather as a - meta-TLD used to look up addresses, and for other purposes. - </description> - </tld> - </itlds><!-- Generic Top Level Domains --> - <gtlds> - <!-- - The following gTLDs are in actual use - --> - - <tld domain="aero"> - <status>SPONSORED</status> - <description>for the air transport industry</description> - </tld> - - <tld domain="biz"> - <status>UNSPONSORED</status> - <description>for business use</description> - </tld> - - <tld domain="cat"> - <status>SPONSORED</status> - <description>for Catalan language/culture</description> - </tld> - - <tld domain="com"> - <status>UNSPONSORED</status> - <description> - for commercial organizations, but unrestricted - </description> - </tld> - - <tld domain="coop"> - <status>SPONSORED</status> - <description>for cooperatives</description> - </tld> - - <tld domain="edu"> - <status>UNSPONSORED</status> - <boost>1.0</boost> - <description> - for post-secondary educational establishments - </description> - </tld> - - <tld domain="gov"> - <status>UNSPONSORED</status> - <description> - for governments and their agencies in the United States - </description> - </tld> - - <tld domain="info"> - <status>UNSPONSORED</status> - <description> - for informational sites, but unrestricted - </description> - </tld> - - <tld domain="int"> - <status>UNSPONSORED</status> - <description> - for international organizations established by treaty - </description> - </tld> - - <tld domain="jobs"> - <status>SPONSORED</status> - <description>for employment-related sites</description> - </tld> - - <tld domain="mil"> - <status>UNSPONSORED</status> - <description>for the US military</description> - </tld> - - <tld domain="mobi"> - <status>SPONSORED</status> - <description>for sites catering to mobile devices</description> - </tld> - - <tld domain="museum"> - <status>SPONSORED</status> - <description>for museums</description> - </tld> - - <tld domain="name"> - <status>UNSPONSORED</status> - <description>for families and individuals</description> - </tld> - - <tld domain="net"> - <status>UNSPONSORED</status> - <description> - originally for network infrastructures, now unrestricted - </description> - </tld> - - <tld domain="org"> - <status>UNSPONSORED</status> - <description> - originally for organizations not clearly falling within the - other gTLDs, now unrestricted - </description> - </tld> - - <tld domain="pro"> - <status>SPONSORED</status> - <description>for certain professions</description> - </tld> - - <tld domain="travel"> - <status>SPONSORED</status> - <description> - for travel agents, airlines, hoteliers, tourism bureaus, etc. - </description> - </tld> - - <!-- - The following gTLDs are in the process of being approved, - and may be added to the root nameservers in the near future - --> - - <tld domain="asia"> - <status>STARTUP</status> - <description>for the Asian community</description> - </tld> - - <tld domain="post"> - <status>PROPOSED</status> - <description>for postal services</description> - </tld> - - <tld domain="tel"> - <status>STARTUP</status> - <description> - for services involving connections between the telephone - network and the Internet - </description> - </tld> - - <tld domain="geo"> - <status>PROPOSED</status> - <description>for geographically related sites</description> - </tld> - - <tld domain="gal"> - <status>PROPOSED</status> - <description>for Galicia, a country within Spain</description> - </tld> - - <tld domain="cym"> - <status>PROPOSED</status> - <description>for Wales, a country within the UK</description> - </tld> - - <tld domain="sco"> - <status>PROPOSED</status> - <description>for Scotland, a country within the UK</description> - </tld> - - <tld domain="kid"> - <status>PROPOSED</status> - <description>for websites designed for children</description> - </tld> - - <tld domain="kids"> - <status>PROPOSED</status> - <description>for websites designed for children</description> - </tld> - - <tld domain="mail"> - <status>PROPOSED</status> - <description>http://en.wikipedia.org/wiki/.mail</description> - </tld> - - <tld domain="web"> - <status>PROPOSED</status> - <description>For Web sites of all sorts</description> - </tld> - - <tld domain="xxx"> - <status>PROPOSED</status> - <description>For Adult entertainment sites</description> - </tld> - - <!-- - The following gTLDs are removed from the registry - --> - <tld domain="nato"> - <status>DELETED</status> - <description> - for NATO sites and operations. Replaced by .int - </description> - </tld> - - <!-- - The following gTLDs are PSEUDO_DOMAINs - --> - <tld domain="bitnet"> - <status>PSEUDO_DOMAIN</status> - <description> - identifying a hostname not connected directly to the Internet, - but a bitnet network - </description> - </tld> - - <tld domain="csnet"> - <status>PSEUDO_DOMAIN</status> - <description> - identifying a hostname not connected directly to the Internet, - but a csnet network - </description> - </tld> - - <tld domain="uucp"> - <status>PSEUDO_DOMAIN</status> - <description> - identifying a hostname not connected directly to the Internet, - but a bitnet network - </description> - </tld> - - <tld domain="local"> - <status>PSEUDO_DOMAIN</status> - <description> - .local is a pseudo top-level domain used by Apple, Inc.'s - Bonjour protocol. - </description> - </tld> - - <tld domain="internal"> - <status>PSEUDO_DOMAIN</status> - <description>alias of .local</description> - </tld> - - <tld domain="onion"> - <status>PSEUDO_DOMAIN</status> - <description> - designates an anonymous or pseudonymous address reachable via - the Tor network. - </description> - </tld> - </gtlds><!-- Country Code Top Level Domains --> - - <cctlds> - <tld domain="ac"> - <country>Ascension Island</country> - </tld> - - <tld domain="ad"> - <country>Andorra</country> - </tld> - - <tld domain="ae"> - <country>United Arab Emirates</country> - </tld> - - <tld domain="af"> - <country>Afghanistan</country> - </tld> - - <tld domain="ag"> - <country>Antigua and Barbuda</country> - </tld> - - <tld domain="ai"> - <country>Anguilla</country> - </tld> - - <tld domain="al"> - <country>Albania</country> - </tld> - - <tld domain="am"> - <country>Armenia</country> - </tld> - - <tld domain="an"> - <country>Netherlands Antilles</country> - </tld> - - <tld domain="ao"> - <country>Angola</country> - </tld> - - <tld domain="aq"> - <country>Antarctica</country> - </tld> - - <tld domain="ar"> - <country>Argentina</country> - </tld> - - <tld domain="as"> - <country>American Samoa</country> - </tld> - - <tld domain="at"> - <country>Austria</country> - </tld> - - <tld domain="au"> - <country>Australia</country> - </tld> - - <tld domain="aw"> - <country>Aruba</country> - </tld> - - <tld domain="ax"> - <country>Aland Islands</country> - </tld> - - <tld domain="az"> - <country>Azerbaijan</country> - </tld> - - <tld domain="ba"> - <country>Bosnia and Herzegovina</country> - </tld> - - <tld domain="bb"> - <country>Barbados</country> - </tld> - - <tld domain="bd"> - <country>Bangladesh</country> - </tld> - - <tld domain="be"> - <country>Belgium</country> - </tld> - - <tld domain="bf"> - <country>Burkina Faso</country> - </tld> - - <tld domain="bg"> - <country>Bulgaria</country> - </tld> - - <tld domain="bh"> - <country>Bahrain</country> - </tld> - - <tld domain="bi"> - <country>Burundi</country> - </tld> - - <tld domain="bj"> - <country>Benin</country> - </tld> - - <tld domain="bm"> - <country>Bermuda</country> - </tld> - - <tld domain="bn"> - <country>Brunei</country> - </tld> - - <tld domain="bo"> - <country>Bolivia</country> - </tld> - - <tld domain="br"> - <country>Brazil</country> - </tld> - - <tld domain="bs"> - <country>Bahamas</country> - </tld> - - <tld domain="bt"> - <country>Bhutan</country> - </tld> - - <tld domain="bu"> - <country>Burma</country> - <status>NOT_IN_USE</status> - <description> - not in use since re-naming of country to Myanmar, see .mm - </description> - </tld> - - <tld domain="bv"> - <country>Bouvet Island</country> - <status>NOT_IN_USE</status> - <description>not in use; no registrations</description> - </tld> - - <tld domain="bw"> - <country>Botswana</country> - </tld> - - <tld domain="by"> - <country>Belarus</country> - </tld> - - <tld domain="bz"> - <country>Belize</country> - </tld> - - <tld domain="ca"> - <country>Canada</country> - </tld> - - <tld domain="cc"> - <country>Cocos Keeling Islands</country> - </tld> - - <tld domain="cd"> - <country>Democratic Republic of the Congo</country> - <description>formerly .zr - Zaire</description> - </tld> - - <tld domain="cf"> - <country>Central African Republic</country> - </tld> - - <tld domain="cg"> - <country>Republic of the Congo</country> - </tld> - - <tld domain="ch"> - <country>Switzerland</country> - </tld> - - <tld domain="ci"> - <country>CÃŽte d'Ivoire</country> - <description>Ivory Coast</description> - </tld> - - <tld domain="ck"> - <country>Cook Islands</country> - </tld> - - <tld domain="cl"> - <country>Chile</country> - </tld> - - <tld domain="cm"> - <country>Cameroon</country> - </tld> - - <tld domain="cn"> - <country>People s Republic of China</country> - </tld> - - <tld domain="co"> - <country>Colombia</country> - </tld> - - <tld domain="cr"> - <country>Costa Rica</country> - </tld> - - <tld domain="cs"> - <country>Serbia and Montenegro</country> - <status>DELETED</status> - <description> - formerly .yu - Yugoslavia; description: on June 3, 2006, - Montenegro declared independence, thus dissolving the state - union) (.cs code not assigned; no DNS) (.cs code previously - used for Czechoslovakia - </description> - </tld> - - <tld domain="cu"> - <country>Cuba</country> - </tld> - - <tld domain="cv"> - <country>Cape Verde</country> - </tld> - - <tld domain="cx"> - <country>Christmas Island</country> - </tld> - - <tld domain="cy"> - <country>Cyprus</country> - </tld> - - <tld domain="cz"> - <country>Czech Republic</country> - </tld> - - <tld domain="dd"> - <country>German Democratic Republic(East Germany)</country> - <status>DELETED</status> - <description>deleted in 1990</description> - </tld> - - <tld domain="de"> - <country>Germany</country> - </tld> - - <tld domain="dj"> - <country>Djibouti</country> - </tld> - - <tld domain="dk"> - <country>Denmark</country> - </tld> - - <tld domain="dm"> - <country>Dominica</country> - </tld> - - <tld domain="do"> - <country>Dominican Republic</country> - </tld> - - <tld domain="dz"> - <country>Algeria</country> - </tld> - - <tld domain="ec"> - <country>Ecuador</country> - </tld> - - <tld domain="ee"> - <country>Estonia</country> - </tld> - - <tld domain="eg"> - <country>Egypt</country> - </tld> - - <tld domain="eh"> - <country>Western Sahara</country> - <status>NOT_IN_USE</status> - <description>not assigned; no DNS</description> - </tld> - - <tld domain="er"> - <country>Eritrea</country> - </tld> - - <tld domain="es"> - <country>Spain</country> - </tld> - - <tld domain="et"> - <country>Ethiopia</country> - </tld> - - <tld domain="eu"> - <country>European Union</country> - <description> - code "exceptionally reserved" by ISO 3166-1 - </description> - </tld> - - <tld domain="fi"> - <country>Finland</country> - </tld> - - <tld domain="fj"> - <country>Fiji</country> - </tld> - - <tld domain="fk"> - <country>Falkland Islands</country> - </tld> - - <tld domain="fm"> - <country>Federated States of Micronesia</country> - </tld> - - <tld domain="fo"> - <country>Faroe Islands</country> - </tld> - - <tld domain="fr"> - <country>France</country> - </tld> - - <tld domain="ga"> - <country>Gabon</country> - </tld> - - <tld domain="gb"> - <country>United Kingdom</country> - <description> - Reserved domain by IANA; deprecated – see .uk - </description> - </tld> - - <tld domain="gd"> - <country>Grenada</country> - </tld> - - <tld domain="ge"> - <country>Georgia</country> - </tld> - - <tld domain="gf"> - <country>French Guiana</country> - </tld> - - <tld domain="gg"> - <country>Guernsey</country> - </tld> - - <tld domain="gh"> - <country>Ghana</country> - </tld> - - <tld domain="gi"> - <country>Gibraltar</country> - </tld> - - <tld domain="gl"> - <country>Greenland</country> - </tld> - - <tld domain="gm"> - <country>Gambia</country> - </tld> - - <tld domain="gn"> - <country>Guinea</country> - </tld> - - <tld domain="gp"> - <country>Guadeloupe</country> - </tld> - - <tld domain="gq"> - <country>Equatorial Guinea</country> - </tld> - - <tld domain="gr"> - <country>Greece</country> - </tld> - - <tld domain="gs"> - <country>South Georgia and the South Sandwich Islands</country> - </tld> - - <tld domain="gt"> - <country>Guatemala</country> - </tld> - - <tld domain="gu"> - <country>Guam</country> - </tld> - - <tld domain="gw"> - <country>Guinea Bissau</country> - </tld> - - <tld domain="gy"> - <country>Guyana</country> - </tld> - - <tld domain="hk"> - <country>Hong Kong</country> - </tld> - - <tld domain="hm"> - <country>Heard Island and McDonald Islands</country> - </tld> - - <tld domain="hn"> - <country>Honduras</country> - </tld> - - <tld domain="hr"> - <country>Croatia</country> - </tld> - - <tld domain="ht"> - <country>Haiti</country> - </tld> - - <tld domain="hu"> - <country>Hungary</country> - </tld> - - <tld domain="id"> - <country>Indonesia</country> - </tld> - - <tld domain="ie"> - <country>Ireland</country> - </tld> - - <tld domain="il"> - <country>Israel</country> - </tld> - - <tld domain="im"> - <country>Isle of Man</country> - </tld> - - <tld domain="in"> - <country>India</country> - </tld> - - <tld domain="io"> - <country>British Indian Ocean Territory</country> - </tld> - - <tld domain="iq"> - <country>Iraq</country> - </tld> - - <tld domain="ir"> - <country>Iran</country> - </tld> - - <tld domain="is"> - <country>Iceland</country> - </tld> - - <tld domain="it"> - <country>Italy</country> - </tld> - - <tld domain="je"> - <country>Jersey</country> - </tld> - - <tld domain="jm"> - <country>Jamaica</country> - </tld> - - <tld domain="jo"> - <country>Jordan</country> - </tld> - - <tld domain="jp"> - <country>Japan</country> - </tld> - - <tld domain="ke"> - <country>Kenya</country> - </tld> - - <tld domain="kg"> - <country>Kyrgyzstan</country> - </tld> - - <tld domain="kh"> - <country>Cambodia</country> - </tld> - - <tld domain="ki"> - <country>Kiribati</country> - </tld> - - <tld domain="km"> - <country>Comoros</country> - </tld> - - <tld domain="kn"> - <country>Saint Kitts and Nevis</country> - </tld> - - <tld domain="kp"> - <country>North Korea</country> - <status>NOT_IN_USE</status> - - <description>not assigned; no DNS</description> - </tld> - - <tld domain="kr"> - <country>South Korea</country> - </tld> - - <tld domain="kw"> - <country>Kuwait</country> - </tld> - - <tld domain="ky"> - <country>Cayman Islands</country> - </tld> - - <tld domain="kz"> - <country>Kazakhstan</country> - </tld> - - <tld domain="la"> - <country>Laos</country> - </tld> - - <tld domain="lb"> - <country>Lebanon</country> - </tld> - - <tld domain="lc"> - <country>Saint Lucia</country> - </tld> - - <tld domain="li"> - <country>Liechtenstein</country> - </tld> - - <tld domain="lk"> - <country>Sri Lanka</country> - </tld> - - <tld domain="lr"> - <country>Liberia</country> - </tld> - - <tld domain="ls"> - <country>Lesotho</country> - </tld> - - <tld domain="lt"> - <country>Lithuania</country> - </tld> - - <tld domain="lu"> - <country>Luxembourg</country> - </tld> - - <tld domain="lv"> - <country>Latvia</country> - </tld> - - <tld domain="ly"> - <country>Libya</country> - </tld> - - <tld domain="ma"> - <country>Morocco</country> - </tld> - - <tld domain="mc"> - <country>Monaco</country> - </tld> - - <tld domain="md"> - <country>Moldova</country> - </tld> - - <tld domain="me"> - <country>Montenegro</country> - </tld> - - <tld domain="mg"> - <country>Madagascar</country> - </tld> - - <tld domain="mh"> - <country>Marshall Islands</country> - </tld> - - <tld domain="mk"> - <country>Republic of Macedonia</country> - </tld> - - <tld domain="ml"> - <country>Mali</country> - </tld> - - <tld domain="mm"> - <country>Myanmar</country> - <description>formerly .bu - Burma</description> - </tld> - - <tld domain="mn"> - <country>Mongolia</country> - </tld> - - <tld domain="mo"> - <country>Macau</country> - </tld> - - <tld domain="mp"> - <country>Northern Mariana Islands</country> - </tld> - - <tld domain="mq"> - <country>Martinique</country> - </tld> - - <tld domain="mr"> - <country>Mauritania</country> - </tld> - - <tld domain="ms"> - <country>Montserrat</country> - </tld> - - <tld domain="mt"> - <country>Malta</country> - </tld> - - <tld domain="mu"> - <country>Mauritius</country> - </tld> - - <tld domain="mv"> - <country>Maldives</country> - </tld> - - <tld domain="mw"> - <country>Malawi</country> - </tld> - - <tld domain="mx"> - <country>Mexico</country> - </tld> - - <tld domain="my"> - <country>Malaysia</country> - </tld> - - <tld domain="mz"> - <country>Mozambique</country> - </tld> - - <tld domain="na"> - <country>Namibia</country> - </tld> - - <tld domain="nc"> - <country>New Caledonia</country> - </tld> - - <tld domain="ne"> - <country>Niger</country> - </tld> - - <tld domain="nf"> - <country>Norfolk Island</country> - </tld> - - <tld domain="ng"> - <country>Nigeria</country> - </tld> - - <tld domain="ni"> - <country>Nicaragua</country> - </tld> - - <tld domain="nl"> - <country>Netherlands</country> - </tld> - - <tld domain="no"> - <country>Norway</country> - </tld> - - <tld domain="np"> - <country>Nepal</country> - </tld> - - <tld domain="nr"> - <country>Nauru</country> - </tld> - - <tld domain="nu"> - <country>Niue</country> - </tld> - - <tld domain="nz"> - <country>New Zealand</country> - </tld> - - <tld domain="om"> - <country>Oman</country> - </tld> - - <tld domain="pa"> - <country>Panama</country> - </tld> - - <tld domain="pe"> - <country>Peru</country> - </tld> - - <tld domain="pf"> - <country>French Polynesia</country> - </tld> - - <tld domain="pg"> - <country>Papua New Guinea</country> - </tld> - - <tld domain="ph"> - <country>Philippines</country> - </tld> - - <tld domain="pk"> - <country>Pakistan</country> - </tld> - - <tld domain="pl"> - <country>Poland</country> - </tld> - - <tld domain="pm"> - <country>Saint Pierre and Miquelon</country> - </tld> - - <tld domain="pn"> - <country>Pitcairn Islands</country> - </tld> - - <tld domain="pr"> - <country>Puerto Rico</country> - </tld> - - <tld domain="ps"> - <country>Palestinian territories</country> - </tld> - - <tld domain="pt"> - <country>Portugal</country> - </tld> - - <tld domain="pw"> - <country>Palau</country> - </tld> - - <tld domain="py"> - <country>Paraguay</country> - </tld> - - <tld domain="qa"> - <country>Qatar</country> - </tld> - - <tld domain="re"> - <country>Réunion</country> - </tld> - - <tld domain="ro"> - <country>Romania</country> - </tld> - - <tld domain="rs"> - <country>Serbia</country> - </tld> - - <tld domain="ru"> - <country>Russia</country> - </tld> - - <tld domain="rw"> - <country>Rwanda</country> - </tld> - - <tld domain="sa"> - <country>Saudi Arabia</country> - </tld> - - <tld domain="sb"> - <country>Solomon Islands</country> - </tld> - - <tld domain="sc"> - <country>Seychelles</country> - </tld> - - <tld domain="sd"> - <country>Sudan</country> - </tld> - - <tld domain="se"> - <country>Sweden</country> - </tld> - - <tld domain="sg"> - <country>Singapore</country> - </tld> - - <tld domain="sh"> - <country>Saint Helena</country> - </tld> - - <tld domain="si"> - <country>Slovenia</country> - </tld> - - <tld domain="sj"> - <country>Svalbard and Jan Mayen Islands</country> - <status>NOT_IN_USE</status> - <description>not in use; no registrations</description> - </tld> - - <tld domain="sk"> - <country>Slovakia</country> - </tld> - - <tld domain="sl"> - <country>Sierra Leone</country> - </tld> - - <tld domain="sm"> - <country>San Marino</country> - </tld> - - <tld domain="sn"> - <country>Senegal</country> - </tld> - - <tld domain="so"> - <country>Somalia</country> - </tld> - - <tld domain="sr"> - <country>Suriname</country> - </tld> - - <tld domain="st"> - <country>São Tomé and PrÃÂncipe</country> - </tld> - - <tld domain="su"> - <country>Soviet Union</country> - <status>DELETED</status> - <description> - deprecated; being phased out; code "transitionally reserved" - by ISO 3166-1 - </description> - </tld> - - <tld domain="sv"> - <country>El Salvador</country> - </tld> - - <tld domain="sy"> - <country>Syria</country> - </tld> - - <tld domain="sz"> - <country>Swaziland</country> - </tld> - - <tld domain="tc"> - <country>Turks and Caicos Islands</country> - </tld> - - <tld domain="td"> - <country>Chad</country> - </tld> - - <tld domain="tf"> - <country>French Southern Territories</country> - </tld> - - <tld domain="tg"> - <country>Togo</country> - </tld> - - <tld domain="th"> - <country>Thailand</country> - </tld> - - <tld domain="tj"> - <country>Tajikistan</country> - </tld> - - <tld domain="tk"> - <country>Tokelau</country> - </tld> - - <tld domain="tl"> - <country>East Timor</country> - <description>formerly .tp</description> - </tld> - - <tld domain="tm"> - <country>Turkmenistan</country> - </tld> - - <tld domain="tn"> - <country>Tunisia</country> - </tld> - - <tld domain="to"> - <country>Tonga</country> - </tld> - - <tld domain="tp"> - <country>East Timor</country> - <status>DELETED</status> - <description> - deprecated - use .tl; code "transitionally reserved" by ISO - 3166-1 - </description> - </tld> - - <tld domain="tr"> - <country>Turkey</country> - </tld> - - <tld domain="tt"> - <country>Trinidad and Tobago</country> - </tld> - - <tld domain="tv"> - <country>Tuvalu</country> - </tld> - - <tld domain="tw"> - <country>Republic of China</country> - <description>Taiwan</description> - </tld> - - <tld domain="tz"> - <country>Tanzania</country> - </tld> - - <tld domain="ua"> - <country>Ukraine</country> - </tld> - - <tld domain="ug"> - <country>Uganda</country> - </tld> - - <tld domain="uk"> - <country>United Kingdom</country> - <description> - code "exceptionally reserved" by ISO 3166-1 (see also .gb) - </description> - </tld> - - <tld domain="um"> - <country>United States Minor Outlying Islands</country> - <status>DELETED</status> - <description>see http://en.wikipedia.org/wiki/.um</description> - </tld> - - <tld domain="us"> - <country>United States</country> - </tld> - - <tld domain="uy"> - <country>Uruguay</country> - </tld> - - <tld domain="uz"> - <country>Uzbekistan</country> - </tld> - - <tld domain="va"> - <country>Vatican City</country> - </tld> - - <tld domain="vc"> - <country>Saint Vincent and the Grenadines</country> - </tld> - - <tld domain="ve"> - <country>Venezuela</country> - </tld> - - <tld domain="vg"> - <country>British Virgin Islands</country> - </tld> - - <tld domain="vi"> - <country>United States Virgin Islands</country> - </tld> - - <tld domain="vn"> - <country>Vietnam</country> - </tld> - - <tld domain="vu"> - <country>Vanuatu</country> - </tld> - - <tld domain="wf"> - <country>Wallis and Futuna</country> - </tld> - - <tld domain="ws"> - <country>Samoa</country> - <description>formerly Western Samoa</description> - </tld> - - <tld domain="ye"> - <country>Yemen</country> - </tld> - - <tld domain="yt"> - <country>Mayotte</country> - </tld> - - <tld domain="yu"> - <country>Yugoslavia</country> - <description> - subsequently renamed Serbia and Montenegro (code officially - replaced by .cs (see above) but still used; code - "transitionally reserved" by ISO 3166-1) - </description> - </tld> - - <tld domain="za"> - <country>South Africa</country> - </tld> - - <tld domain="zm"> - <country>Zambia</country> - </tld> - - <tld domain="zr"> - <country>Zaire</country> - <status>DELETED</status> - <description>replaced by .cd</description> - </tld> - - <tld domain="zw"> - <country>Zimbabwe</country> - </tld> - </cctlds> - </tlds> - - <!-- Second Level Domains --> - <suffixes> - - <!-- .us Second Level Domains --> - <suffix domain="as.us" /> - <suffix domain="gu.us" /> - <suffix domain="pr.us" /> - <suffix domain="vi.us" /> - - <!-- .uk Second Level Domains --> - <suffix domain="ac.uk" /> - <suffix domain="co.uk" /> - <suffix domain="gov.uk" /> - <suffix domain="ltd.uk" /> - <suffix domain="me.uk" /> - <suffix domain="mod.uk" /> - <suffix domain="net.uk" /> - <suffix domain="nic.uk" /> - <suffix domain="nhs.uk" /> - <suffix domain="org.uk" /> - <suffix domain="pic.uk" /> - <suffix domain="police.uk" /> - <suffix domain="govt.uk"> - <status>DELETED</status> - </suffix> - <suffix domain="orgn.uk"> - <status>DELETED</status> - </suffix> - <suffix domain="mil.uk"> - <status>DELETED</status> - </suffix> - <suffix domain="lea.uk"> - <status>DELETED</status> - </suffix> - - - <!-- .tr Second Level Domains --> - <suffix domain="com.tr" /> - <suffix domain="gen.tr" /> - <suffix domain="org.tr"/> - <suffix domain="biz.tr" /> - <suffix domain="info.tr" /> - <suffix domain="av.tr" /> - <suffix domain="dr.tr" /> - <suffix domain="pol.tr" /> - <suffix domain="bel.tr" /> - <suffix domain="mil.tr" /> - <suffix domain="mil.tr" /> - <suffix domain="bbs.tr" /> - <suffix domain="k12.tr" /> - <suffix domain="edu.tr"/> - <suffix domain="name.tr" /> - <suffix domain="net.tr" /> - <suffix domain="gov.tr" /> - <suffix domain="gov.tr" /> - <suffix domain="web.tr" /> - <suffix domain="tel.tr" /> - <suffix domain="tv.tr" /> - - <!-- .hk Second Level Domains --> - <suffix domain="ust.hk" /> - <suffix domain="hku.hk" /> - <suffix domain="cuhk.hk" /> - - <!-- .it Second Level Domains --> - - <suffix domain="edu.it" /> - <suffix domain="gov.it" /> - <suffix domain="italy.it" /> - - <!-- .az Second Level Domains --> - <suffix domain="pro.az" /> - - <!-- .au Second Level Domains --> - - <suffix domain="com.au" /> - <suffix domain="net.au" /> - <suffix domain="gov.au" /> - <suffix domain="org.au" /> - <suffix domain="edu.au" /> - <suffix domain="csiro.au" /> - <suffix domain="asn.au" /> - <suffix domain="id.au" /> - <!-- New second level domains --> - <suffix domain="act.au" /> - <suffix domain="nsw.au" /> - <suffix domain="nt.au" /> - <suffix domain="qld.au" /> - <suffix domain="sa.au" /> - <suffix domain="tas.au" /> - <suffix domain="vic.au" /> - <suffix domain="wa.au" /> - - <!-- .int Second Level Domains --> - <suffix domain="eu.int" /> - - <!-- .br Second Level Domains --> - <suffix domain="adm.br" /> - <suffix domain="adv.br" /> - <suffix domain="agr.br" /> - <suffix domain="am.br" /> - <suffix domain="arq.br" /> - <suffix domain="art.br" /> - <suffix domain="ato.br" /> - <suffix domain="bio.br" /> - <suffix domain="blog.br" /> - <suffix domain="bmd.br" /> - <suffix domain="cim.br" /> - <suffix domain="cng.br" /> - <suffix domain="cnt.br" /> - <suffix domain="com.br" /> - <suffix domain="coop.br" /> - <suffix domain="ecn.br" /> - <suffix domain="edu.br" /> - <suffix domain="eng.br" /> - <suffix domain="esp.br" /> - <suffix domain="etc.br" /> - <suffix domain="eti.br" /> - <suffix domain="far.br" /> - <suffix domain="flog.br" /> - <suffix domain="fm.br" /> - <suffix domain="fnd.br" /> - <suffix domain="fot.br" /> - <suffix domain="fst.br" /> - <suffix domain="g12.br" /> - <suffix domain="ggf.br" /> - <suffix domain="gov.br" /> - <suffix domain="imb.br" /> - <suffix domain="ind.br" /> - <suffix domain="inf.br" /> - <suffix domain="jor.br" /> - <suffix domain="lel.br" /> - <suffix domain="mat.br" /> - <suffix domain="med.br" /> - <suffix domain="mil.br" /> - <suffix domain="mus.br" /> - <suffix domain="net.br" /> - <suffix domain="nom.br" /> - <suffix domain="not.br" /> - <suffix domain="ntr.br" /> - <suffix domain="odo.br" /> - <suffix domain="org.br" /> - <suffix domain="ppg.br" /> - <suffix domain="pro.br" /> - <suffix domain="psc.br" /> - <suffix domain="psi.br" /> - <suffix domain="qsl.br" /> - <suffix domain="rec.br" /> - <suffix domain="slg.br" /> - <suffix domain="srv.br" /> - <suffix domain="tmp.br" /> - <suffix domain="trd.br" /> - <suffix domain="tur.br" /> - <suffix domain="tv.br" /> - <suffix domain="vet.br" /> - <suffix domain="vlog.br" /> - <suffix domain="wiki.br" /> - <suffix domain="zlg.br" /> - - <!-- - Below elements are generated from the file - at http://publicsuffix.org/. - see org.apache.nutch.util.domain.MozillaPublicSuffixListParser.java - --> - - <!-- ac : http://en.wikipedia.org/wiki/.am--> - <suffix domain="com.ac" /> - <suffix domain="edu.ac" /> - <suffix domain="gov.ac" /> - <suffix domain="net.ac" /> - <suffix domain="mil.ac" /> - <suffix domain="org.ac" /> - - <!-- ad : http://en.wikipedia.org/wiki/.ad--> - <suffix domain="nom.ad" /> - - <!-- ae : http://en.wikipedia.org/wiki/.ae--> - <suffix domain="net.ae" /> - <suffix domain="gov.ae" /> - <suffix domain="ac.ae" /> - <suffix domain="sch.ae" /> - <suffix domain="org.ae" /> - <suffix domain="mil.ae" /> - <suffix domain="pro.ae" /> - <suffix domain="name.ae" /> - - <!-- aero : see http://www.information.aero/index.php?id=66--> - <suffix domain="accident-investigation.aero" /> - <suffix domain="accident-prevention.aero" /> - <suffix domain="aerobatic.aero" /> - <suffix domain="aeroclub.aero" /> - <suffix domain="aerodrome.aero" /> - <suffix domain="agents.aero" /> - <suffix domain="aircraft.aero" /> - <suffix domain="airline.aero" /> - <suffix domain="airport.aero" /> - <suffix domain="air-surveillance.aero" /> - <suffix domain="airtraffic.aero" /> - <suffix domain="air-traffic-control.aero" /> - <suffix domain="ambulance.aero" /> - <suffix domain="amusement.aero" /> - <suffix domain="association.aero" /> - <suffix domain="author.aero" /> - <suffix domain="ballooning.aero" /> - <suffix domain="broker.aero" /> - <suffix domain="caa.aero" /> - <suffix domain="cargo.aero" /> - <suffix domain="catering.aero" /> - <suffix domain="certification.aero" /> - <suffix domain="championship.aero" /> - <suffix domain="charter.aero" /> - <suffix domain="civilaviation.aero" /> - <suffix domain="club.aero" /> - <suffix domain="conference.aero" /> - <suffix domain="consultant.aero" /> - <suffix domain="consulting.aero" /> - <suffix domain="control.aero" /> - <suffix domain="council.aero" /> - <suffix domain="crew.aero" /> - <suffix domain="design.aero" /> - <suffix domain="dgca.aero" /> - <suffix domain="educator.aero" /> - <suffix domain="emergency.aero" /> - <suffix domain="engine.aero" /> - <suffix domain="engineer.aero" /> - <suffix domain="entertainment.aero" /> - <suffix domain="equipment.aero" /> - <suffix domain="exchange.aero" /> - <suffix domain="express.aero" /> - <suffix domain="federation.aero" /> - <suffix domain="flight.aero" /> - <suffix domain="freight.aero" /> - <suffix domain="fuel.aero" /> - <suffix domain="gliding.aero" /> - <suffix domain="government.aero" /> - <suffix domain="groundhandling.aero" /> - <suffix domain="group.aero" /> - <suffix domain="hanggliding.aero" /> - <suffix domain="homebuilt.aero" /> - <suffix domain="insurance.aero" /> - <suffix domain="journal.aero" /> - <suffix domain="journalist.aero" /> - <suffix domain="leasing.aero" /> - <suffix domain="logistics.aero" /> - <suffix domain="magazine.aero" /> - <suffix domain="maintenance.aero" /> - <suffix domain="marketplace.aero" /> - <suffix domain="media.aero" /> - <suffix domain="microlight.aero" /> - <suffix domain="modelling.aero" /> - <suffix domain="navigation.aero" /> - <suffix domain="parachuting.aero" /> - <suffix domain="paragliding.aero" /> - <suffix domain="passenger-association.aero" /> - <suffix domain="pilot.aero" /> - <suffix domain="press.aero" /> - <suffix domain="production.aero" /> - <suffix domain="recreation.aero" /> - <suffix domain="repbody.aero" /> - <suffix domain="res.aero" /> - <suffix domain="research.aero" /> - <suffix domain="rotorcraft.aero" /> - <suffix domain="safety.aero" /> - <suffix domain="scientist.aero" /> - <suffix domain="services.aero" /> - <suffix domain="show.aero" /> - <suffix domain="skydiving.aero" /> - <suffix domain="software.aero" /> - <suffix domain="student.aero" /> - <suffix domain="taxi.aero" /> - <suffix domain="trader.aero" /> - <suffix domain="trading.aero" /> - <suffix domain="trainer.aero" /> - <suffix domain="union.aero" /> - <suffix domain="workinggroup.aero" /> - <suffix domain="works.aero" /> - - <!-- af : http://www.nic.af/help.jsp--> - <suffix domain="gov.af" /> - <suffix domain="com.af" /> - <suffix domain="org.af" /> - <suffix domain="net.af" /> - <suffix domain="edu.af" /> - - <!-- ag : http://www.nic.ag/prices.htm--> - <suffix domain="com.ag" /> - <suffix domain="org.ag" /> - <suffix domain="net.ag" /> - <suffix domain="co.ag" /> - <suffix domain="nom.ag" /> - - <!-- ai : http://nic.com.ai/--> - <suffix domain="off.ai" /> - <suffix domain="com.ai" /> - <suffix domain="net.ai" /> - <suffix domain="org.ai" /> - - <!-- al : http://www.inima.al/Domains.html--> - <suffix domain="gov.al" /> - <suffix domain="edu.al" /> - <suffix domain="org.al" /> - <suffix domain="com.al" /> - <suffix domain="net.al" /> - - <!-- am : http://en.wikipedia.org/wiki/.am--> - - <!-- an : http://www.una.an/an_domreg/default.asp--> - <suffix domain="com.an" /> - <suffix domain="net.an" /> - <suffix domain="org.an" /> - <suffix domain="edu.an" /> - - <!-- ao : http://en.wikipedia.org/wiki/.ao--> - <!-- list of 2nd level TLDs ?--> - - <!-- aq : http://en.wikipedia.org/wiki/.aq--> - - <!-- ar : http://en.wikipedia.org/wiki/.ar--> - <suffix domain="congresodelalengua3.ar" /> - <suffix domain="educ.ar" /> - <suffix domain="gobiernoelectronico.ar" /> - <suffix domain="mecon.ar" /> - <suffix domain="nacion.ar" /> - <suffix domain="nic.ar" /> - <suffix domain="promocion.ar" /> - <suffix domain="retina.ar" /> - <suffix domain="uba.ar" /> - - <!-- arpa : http://en.wikipedia.org/wiki/.arpa--> - <suffix domain="e164.arpa" /> - <suffix domain="in-addr.arpa" /> - <suffix domain="ip6.arpa" /> - <suffix domain="uri.arpa" /> - <suffix domain="urn.arpa" /> - - <!-- as : http://en.wikipedia.org/wiki/.as--> - - <!-- at : http://en.wikipedia.org/wiki/.at--> - <suffix domain="gv.at" /> - <suffix domain="ac.at" /> - <suffix domain="co.at" /> - <suffix domain="or.at" /> - - <!-- au : http://en.wikipedia.org/wiki/.au--> - <!-- au geographical names (vic.au etc... are covered above)--> - <suffix domain="act.edu.au" /> - <suffix domain="nsw.edu.au" /> - <suffix domain="nt.edu.au" /> - <suffix domain="qld.edu.au" /> - <suffix domain="sa.edu.au" /> - <suffix domain="tas.edu.au" /> - <suffix domain="vic.edu.au" /> - <suffix domain="wa.edu.au" /> - <suffix domain="act.gov.au" /> - <suffix domain="nsw.gov.au" /> - <suffix domain="nt.gov.au" /> - <suffix domain="qld.gov.au" /> - <suffix domain="sa.gov.au" /> - <suffix domain="tas.gov.au" /> - <suffix domain="vic.gov.au" /> - <suffix domain="wa.gov.au" /> - - <!-- aw : http://en.wikipedia.org/wiki/.aw--> - <suffix domain="com.aw" /> - - <!-- ax : http://en.wikipedia.org/wiki/.ax--> - - <!-- az : http://en.wikipedia.org/wiki/.az--> - <suffix domain="com.az" /> - <suffix domain="net.az" /> - <suffix domain="int.az" /> - <suffix domain="gov.az" /> - <suffix domain="org.az" /> - <suffix domain="edu.az" /> - <suffix domain="info.az" /> - <suffix domain="pp.az" /> - <suffix domain="mil.az" /> - <suffix domain="name.az" /> - <suffix domain="biz.az" /> - - <!-- ba : http://en.wikipedia.org/wiki/.ba--> - <suffix domain="org.ba" /> - <suffix domain="net.ba" /> - <suffix domain="edu.ba" /> - <suffix domain="gov.ba" /> - <suffix domain="mil.ba" /> - <suffix domain="unsa.ba" /> - <suffix domain="unbi.ba" /> - <suffix domain="co.ba" /> - <suffix domain="com.ba" /> - <suffix domain="rs.ba" /> - - <!-- bb : http://en.wikipedia.org/wiki/.bb--> - <suffix domain="com.bb" /> - <suffix domain="edu.bb" /> - <suffix domain="gov.bb" /> - <suffix domain="net.bb" /> - <suffix domain="org.bb" /> - - <!-- bd : http://en.wikipedia.org/wiki/.bd--> - - <!-- be : http://en.wikipedia.org/wiki/.be--> - <suffix domain="ac.be" /> - - <!-- bf : http://en.wikipedia.org/wiki/.bf--> - - <!-- bg : http://en.wikipedia.org/wiki/.bg--> - - <!-- bh : http://en.wikipedia.org/wiki/.bh--> - <!-- list of 2nd level tlds ?--> - - <!-- bi : http://en.wikipedia.org/wiki/.bi--> - <!-- list of 2nd level tlds ?--> - - <!-- biz : http://en.wikipedia.org/wiki/.biz--> - - <!-- bj : http://en.wikipedia.org/wiki/.bj--> - <!-- list of 2nd level tlds ?--> - - <!-- bm : http://www.bermudanic.bm/dnr-text.txt--> - <suffix domain="com.bm" /> - <suffix domain="edu.bm" /> - <suffix domain="gov.bm" /> - <suffix domain="net.bm" /> - <suffix domain="org.bm" /> - - <!-- bn : http://en.wikipedia.org/wiki/.bn--> - - <!-- bo : http://www.nic.bo/--> - <suffix domain="com.bo" /> - <suffix domain="edu.bo" /> - <suffix domain="gov.bo" /> - <suffix domain="gob.bo" /> - <suffix domain="int.bo" /> - <suffix domain="org.bo" /> - <suffix domain="net.bo" /> - <suffix domain="mil.bo" /> - <suffix domain="tv.bo" /> - - <!-- br : http://en.wikipedia.org/wiki/.br--> - - <!-- bs : http://www.nic.bs/rules.html--> - <suffix domain="com.bs" /> - <suffix domain="net.bs" /> - <suffix domain="org.bs" /> - <suffix domain="edu.bs" /> - <suffix domain="gov.bs" /> - - <!-- bt : http://en.wikipedia.org/wiki/.bt--> - - <!-- bw : http://en.wikipedia.org/wiki/.bw--> - <!-- list of 2nd level tlds ?--> - - <!-- by : http://en.wikipedia.org/wiki/.by--> - <!-- list of 2nd level tlds ?--> - - <!-- bz : http://en.wikipedia.org/wiki/.bz--> - <!-- list of 2nd level tlds ?--> - - <!-- ca : http://en.wikipedia.org/wiki/.ca--> - <!-- ca geographical names--> - <suffix domain="ab.ca" /> - <suffix domain="bc.ca" /> - <suffix domain="mb.ca" /> - <suffix domain="nb.ca" /> - <suffix domain="nf.ca" /> - <suffix domain="nl.ca" /> - <suffix domain="ns.ca" /> - <suffix domain="nt.ca" /> - <suffix domain="nu.ca" /> - <suffix domain="on.ca" /> - <suffix domain="pe.ca" /> - <suffix domain="qc.ca" /> - <suffix domain="sk.ca" /> - <suffix domain="yk.ca" /> - - <!-- cat : http://en.wikipedia.org/wiki/.cat--> - - <!-- cc : http://en.wikipedia.org/wiki/.cc--> - - <!-- cd : http://en.wikipedia.org/wiki/.cd--> - - <!-- cf : http://en.wikipedia.org/wiki/.cf--> - - <!-- cg : http://en.wikipedia.org/wiki/.cg--> - - <!-- ch : http://en.wikipedia.org/wiki/.ch--> - - <!-- ci : http://en.wikipedia.org/wiki/.ci--> - <!-- list of 2nd level tlds ?--> - - <!-- ck : http://en.wikipedia.org/wiki/.ck--> - - <!-- cl : http://en.wikipedia.org/wiki/.cl--> - - <!-- cm : http://en.wikipedia.org/wiki/.cm--> - - <!-- cn : http://en.wikipedia.org/wiki/.cn--> - <suffix domain="ac.cn" /> - <suffix domain="com.cn" /> - <suffix domain="edu.cn" /> - <suffix domain="gov.cn" /> - <suffix domain="net.cn" /> - <suffix domain="org.cn" /> - <!-- cn geographic names--> - <suffix domain="ah.cn" /> - <suffix domain="bj.cn" /> - <suffix domain="cq.cn" /> - <suffix domain="fj.cn" /> - <suffix domain="gd.cn" /> - <suffix domain="gs.cn" /> - <suffix domain="gz.cn" /> - <suffix domain="gx.cn" /> - <suffix domain="ha.cn" /> - <suffix domain="hb.cn" /> - <suffix domain="he.cn" /> - <suffix domain="hi.cn" /> - <suffix domain="hl.cn" /> - <suffix domain="hn.cn" /> - <suffix domain="jl.cn" /> - <suffix domain="js.cn" /> - <suffix domain="jx.cn" /> - <suffix domain="ln.cn" /> - <suffix domain="nm.cn" /> - <suffix domain="nx.cn" /> - <suffix domain="qh.cn" /> - <suffix domain="sc.cn" /> - <suffix domain="sd.cn" /> - <suffix domain="sh.cn" /> - <suffix domain="sn.cn" /> - <suffix domain="sx.cn" /> - <suffix domain="tj.cn" /> - <suffix domain="xj.cn" /> - <suffix domain="xz.cn" /> - <suffix domain="yn.cn" /> - <suffix domain="zj.cn" /> - - <!-- co : http://en.wikipedia.org/wiki/.co--> - - <!-- com : http://en.wikipedia.org/wiki/.com--> - - <!-- coop : http://en.wikipedia.org/wiki/.coop--> - - <!-- cr : http://en.wikipedia.org/wiki/.cr--> - - <!-- cu : http://en.wikipedia.org/wiki/.cu--> - <suffix domain="com.cu" /> - <suffix domain="edu.cu" /> - <suffix domain="org.cu" /> - <suffix domain="net.cu" /> - <suffix domain="gov.cu" /> - <suffix domain="inf.cu" /> - - <!-- cv : http://en.wikipedia.org/wiki/.cv--> - - <!-- cx : http://en.wikipedia.org/wiki/.cx--> - - <!-- cy : http://en.wikipedia.org/wiki/.cy--> - - <!-- cz : http://en.wikipedia.org/wiki/.cz--> - - <!-- de : http://en.wikipedia.org/wiki/.de--> - - <!-- dj : http://en.wikipedia.org/wiki/.dj--> - - <!-- dk : http://en.wikipedia.org/wiki/.dk--> - - <!-- dm : http://en.wikipedia.org/wiki/.dm--> - <suffix domain="com.dm" /> - <suffix domain="net.dm" /> - <suffix domain="org.dm" /> - - <!-- do : http://en.wikipedia.org/wiki/.do--> - - <!-- dz : http://en.wikipedia.org/wiki/.dz--> - <suffix domain="com.dz" /> - <suffix domain="org.dz" /> - <suffix domain="net.dz" /> - <suffix domain="gov.dz" /> - <suffix domain="edu.dz" /> - <suffix domain="asso.dz" /> - <suffix domain="pol.dz" /> - <suffix domain="art.dz" /> - - <!-- ec : http://www.nic.ec/reg/paso1.asp--> - <suffix domain="com.ec" /> - <suffix domain="info.ec" /> - <suffix domain="net.ec" /> - <suffix domain="fin.ec" /> - <suffix domain="med.ec" /> - <suffix domain="pro.ec" /> - <suffix domain="org.ec" /> - <suffix domain="edu.ec" /> - <suffix domain="gov.ec" /> - <suffix domain="mil.ec" /> - - <!-- edu : http://en.wikipedia.org/wiki/.edu--> - - <!-- ee : http://www3.eenet.ee/ee/application.html--> - <suffix domain="com.ee" /> - <suffix domain="org.ee" /> - <suffix domain="fie.ee" /> - <suffix domain="pri.ee" /> - - <!-- eg : http://en.wikipedia.org/wiki/.eg--> - - <!-- er : http://en.wikipedia.org/wiki/.er--> - - <!-- es : https://www.nic.es/site_ingles/ingles/dominios/index.html--> - <suffix domain="com.es" /> - <suffix domain="nom.es" /> - <suffix domain="org.es" /> - <suffix domain="gob.es" /> - <suffix domain="edu.es" /> - - <!-- et : http://en.wikipedia.org/wiki/.et--> - - <!-- eu : http://en.wikipedia.org/wiki/.eu--> - - <!-- fi : http://en.wikipedia.org/wiki/.fi--> - - <!-- fj : http://en.wikipedia.org/wiki/.fj--> - - <!-- fk : http://en.wikipedia.org/wiki/.fk--> - - <!-- fm : http://en.wikipedia.org/wiki/.fm--> - - <!-- fo : http://en.wikipedia.org/wiki/.fo--> - - <!-- fr : http://www.afnic.fr/--> - <!-- domaines descriptifs : http://www.afnic.fr/obtenir/chartes/nommage-fr/annexe-descriptifs--> - <suffix domain="com.fr" /> - <suffix domain="asso.fr" /> - <suffix domain="nom.fr" /> - <suffix domain="prd.fr" /> - <suffix domain="presse.fr" /> - <suffix domain="tm.fr" /> - <!-- domaines sectoriels : http://www.afnic.fr/obtenir/chartes/nommage-fr/annexe-sectoriels--> - <suffix domain="aeroport.fr" /> - <suffix domain="assedic.fr" /> - <suffix domain="avocat.fr" /> - <suffix domain="avoues.fr" /> - <suffix domain="cci.fr" /> - <suffix domain="chambagri.fr" /> - <suffix domain="chirurgiens-dentistes.fr" /> - <suffix domain="experts-comptables.fr" /> - <suffix domain="geometre-expert.fr" /> - <suffix domain="gouv.fr" /> - <suffix domain="greta.fr" /> - <suffix domain="huissier-justice.fr" /> - <suffix domain="medecin.fr" /> - <suffix domain="notaires.fr" /> - <suffix domain="pharmacien.fr" /> - <suffix domain="port.fr" /> - <suffix domain="veterinaire.fr" /> - - <!-- ga : http://en.wikipedia.org/wiki/.ga--> - - <!-- gd : http://en.wikipedia.org/wiki/.gd--> - - <!-- ge : http://www.nic.net.ge/policy_en.pdf--> - <suffix domain="com.ge" /> - <suffix domain="edu.ge" /> - <suffix domain="gov.ge" /> - <suffix domain="org.ge" /> - <suffix domain="mil.ge" /> - <suffix domain="net.ge" /> - <suffix domain="pvt.ge" /> - - <!-- gf : http://en.wikipedia.org/wiki/.gf--> - - <!-- gg : http://www.channelisles.net/tandc.shtml--> - <suffix domain="co.gg" /> - <suffix domain="org.gg" /> - <suffix domain="net.gg" /> - <suffix domain="sch.gg" /> - <suffix domain="gov.gg" /> - - <!-- gh : http://www.ghana.com/domain.htm--> - - <!-- gi : http://www.nic.gi/rules.html--> - <suffix domain="com.gi" /> - <suffix domain="ltd.gi" /> - <suffix domain="gov.gi" /> - <suffix domain="mod.gi" /> - <suffix domain="edu.gi" /> - <suffix domain="org.gi" /> - - <!-- gl : http://en.wikipedia.org/wiki/.gl--> - - <!-- gm : http://www.nic.gm/htmlpages%5Cgm-policy.htm--> - - <!-- gn : http://psg.com/dns/gn/gn.txt--> - - <!-- gov : http://en.wikipedia.org/wiki/.gov--> - - <!-- gp : http://www.nic.gp/index_en.php?url=charte_en.php--> - <suffix domain="com.gp" /> - <suffix domain="net.gp" /> - <suffix domain="edu.gp" /> - <suffix domain="org.gp" /> - - <!-- gq : http://en.wikipedia.org/wiki/.gq--> - - <!-- gr : https://grweb.ics.forth.gr/english/1617-B-2002.html--> - <suffix domain="com.gr" /> - <suffix domain="edu.gr" /> - <suffix domain="net.gr" /> - <suffix domain="org.gr" /> - <suffix domain="gov.gr" /> - - <!-- gs : http://en.wikipedia.org/wiki/.gs--> - - <!-- gt : http://www.gt/politicas.html--> - - <!-- gu : http://gadao.gov.gu/registration.txt--> - - <!-- gw : http://en.wikipedia.org/wiki/.gw--> - - <!-- gy : http://en.wikipedia.org/wiki/.gy--> - - <!-- hk : http://en.wikipedia.org/wiki/.hk--> - <suffix domain="com.hk" /> - <suffix domain="edu.hk" /> - <suffix domain="gov.hk" /> - <suffix domain="idv.hk" /> - <suffix domain="net.hk" /> - <suffix domain="org.hk" /> - - <!-- hm : http://en.wikipedia.org/wiki/.hm--> - - <!-- hn : http://www.nic.hn/politicas/ps02,,05.html--> - <suffix domain="com.hn" /> - <suffix domain="edu.hn" /> - <suffix domain="org.hn" /> - <suffix domain="net.hn" /> - <suffix domain="mil.hn" /> - <suffix domain="gob.hn" /> - - <!-- hr : http://www.dns.hr/documents/pdf/HRTLD-regulations.pdf--> - <suffix domain="iz.hr" /> - <suffix domain="from.hr" /> - <suffix domain="name.hr" /> - <suffix domain="com.hr" /> - - <!-- ht : http://www.nic.ht/info/charte.cfm--> - <suffix domain="com.ht" /> - <suffix domain="shop.ht" /> - <suffix domain="firm.ht" /> - <suffix domain="info.ht" /> - <suffix domain="adult.ht" /> - <suffix domain="net.ht" /> - <suffix domain="pro.ht" /> - <suffix domain="org.ht" /> - <suffix domain="med.ht" /> - <suffix domain="art.ht" /> - <suffix domain="coop.ht" /> - <suffix domain="pol.ht" /> - <suffix domain="asso.ht" /> - <suffix domain="edu.ht" /> - <suffix domain="rel.ht" /> - <suffix domain="gouv.ht" /> - <suffix domain="perso.ht" /> - - <!-- hu : http://www.domain.hu/domain/English/sld.html--> - <suffix domain="co.hu" /> - <suffix domain="info.hu" /> - <suffix domain="org.hu" /> - <suffix domain="priv.hu" /> - <suffix domain="sport.hu" /> - <suffix domain="tm.hu" /> - <suffix domain="2000.hu" /> - <suffix domain="agrar.hu" /> - <suffix domain="bolt.hu" /> - <suffix domain="casino.hu" /> - <suffix domain="city.hu" /> - <suffix domain="erotica.hu" /> - <suffix domain="erotika.hu" /> - <suffix domain="film.hu" /> - <suffix domain="forum.hu" /> - <suffix domain="games.hu" /> - <suffix domain="hotel.hu" /> - <suffix domain="ingatlan.hu" /> - <suffix domain="jogasz.hu" /> - <suffix domain="konyvelo.hu" /> - <suffix domain="lakas.hu" /> - <suffix domain="media.hu" /> - <suffix domain="news.hu" /> - <suffix domain="reklam.hu" /> - <suffix domain="sex.hu" /> - <suffix domain="shop.hu" /> - <suffix domain="suli.hu" /> - <suffix domain="szex.hu" /> - <suffix domain="tozsde.hu" /> - <suffix domain="utazas.hu" /> - <suffix domain="video.hu" /> - - <!-- id : http://en.wikipedia.org/wiki/.id--> - - <!-- ie : http://en.wikipedia.org/wiki/.ie--> - - <!-- il : http://en.wikipedia.org/wiki/.il--> - - <!-- im : https://www.nic.im/pdfs/imfaqs.pdf--> - <suffix domain="co.im" /> - <suffix domain="ltd.co.im" /> - <suffix domain="plc.co.im" /> - <suffix domain="net.im" /> - <suffix domain="gov.im" /> - <suffix domain="org.im" /> - <suffix domain="nic.im" /> - <suffix domain="ac.im" /> - - <!-- in : http://en.wikipedia.org/wiki/.in--> - <suffix domain="co.in" /> - <suffix domain="firm.in" /> - <suffix domain="net.in" /> - <suffix domain="org.in" /> - <suffix domain="gen.in" /> - <suffix domain="ind.in" /> - <suffix domain="nic.in" /> - <suffix domain="ac.in" /> - <suffix domain="edu.in" /> - <suffix domain="res.in" /> - <suffix domain="gov.in" /> - <suffix domain="mil.in" /> - - <!-- info : http://en.wikipedia.org/wiki/.info--> - - <!-- int : http://en.wikipedia.org/wiki/.int--> - - <!-- io : http://www.nic.io/rules.html--> - <!-- list of 2nd level tlds ?--> - - <!-- iq : http://en.wikipedia.org/wiki/.iq--> - <!-- no registrar website found, but google shows .gov.iq and .edu.iq websites--> - <suffix domain="gov.iq" /> - <suffix domain="edu.iq" /> - - <!-- ir : http://www.nic.ir/ascii/Appendix1.htm--> - <suffix domain="ac.ir" /> - <suffix domain="co.ir" /> - <suffix domain="gov.ir" /> - <suffix domain="id.ir" /> - <suffix domain="net.ir" /> - <suffix domain="org.ir" /> - <suffix domain="sch.ir" /> - - <!-- is : http://www.isnic.is/domain/rules.php--> - <suffix domain="net.is" /> - <suffix domain="com.is" /> - <suffix domain="edu.is" /> - <suffix domain="gov.is" /> - <suffix domain="org.is" /> - <suffix domain="int.is" /> - - <!-- it : http://en.wikipedia.org/wiki/.it--> - <suffix domain="gov.edu" /> - <!-- geo-names found at http://www.nic.it/RA/en/domini/regole/nomi-riservati.pdf--> - <suffix domain="agrigento.it" /> - <suffix domain="ag.it" /> - <suffix domain="alessandria.it" /> - <suffix domain="al.it" /> - <suffix domain="ancona.it" /> - <suffix domain="an.it" /> - <suffix domain="aosta.it" /> - <suffix domain="aoste.it" /> - <suffix domain="ao.it" /> - <suffix domain="arezzo.it" /> - <suffix domain="ar.it" /> - <suffix domain="ascoli-piceno.it" /> - <suffix domain="ascolipiceno.it" /> - <suffix domain="ap.it" /> - <suffix domain="asti.it" /> - <suffix domain="at.it" /> - <suffix domain="avellino.it" /> - <suffix domain="av.it" /> - <suffix domain="bari.it" /> - <suffix domain="ba.it" /> - <suffix domain="barlettaandriatrani.it" /> - <suffix domain="barletta-andria-trani.it" /> - <suffix domain="belluno.it" /> - <suffix domain="bl.it" /> - <suffix domain="benevento.it" /> - <suffix domain="bn.it" /> - <suffix domain="bergamo.it" /> - <suffix domain="bg.it" /> - <suffix domain="biella.it" /> - <suffix domain="bi.it" /> - <suffix domain="bologna.it" /> - <suffix domain="bo.it" /> - <suffix domain="bolzano.it" /> - <suffix domain="bozen.it" /> - <suffix domain="balsan.it" /> - <suffix domain="alto-adige.it" /> - <suffix domain="altoadige.it" /> - <suffix domain="suedtirol.it" /> - <suffix domain="bz.it" /> - <suffix domain="brescia.it" /> - <suffix domain="bs.it" /> - <suffix domain="brindisi.it" /> - <suffix domain="br.it" /> - <suffix domain="cagliari.it" /> - <suffix domain="ca.it" /> - <suffix domain="caltanissetta.it" /> - <suffix domain="cl.it" /> - <suffix domain="campobasso.it" /> - <suffix domain="cb.it" /> - <suffix domain="caserta.it" /> - <suffix domain="ce.it" /> - <suffix domain="catania.it" /> - <suffix domain="ct.it" /> - <suffix domain="catanzaro.it" /> - <suffix domain="cz.it" /> - <suffix domain="chieti.it" /> - <suffix domain="ch.it" /> - <suffix domain="como.it" /> - <suffix domain="co.it" /> - <suffix domain="cosenza.it" /> - <suffix domain="cs.it" /> - <suffix domain="cremona.it" /> - <suffix domain="cr.it" /> - <suffix domain="crotone.it" /> - <suffix domain="kr.it" /> - <suffix domain="cuneo.it" /> - <suffix domain="cn.it" /> - <suffix domain="enna.it" /> - <suffix domain="en.it" /> - <suffix domain="fermo.it" /> - <suffix domain="ferrara.it" /> - <suffix domain="fe.it" /> - <suffix domain="firenze.it" /> - <suffix domain="florence.it" /> - <suffix domain="fi.it" /> - <suffix domain="foggia.it" /> - <suffix domain="fg.it" /> - <suffix domain="forli-cesena.it" /> - <suffix domain="forlicesena.it" /> - <suffix domain="fc.it" /> - <suffix domain="frosinone.it" /> - <suffix domain="fr.it" /> - <suffix domain="genova.it" /> - <suffix domain="genoa.it" /> - <suffix domain="ge.it" /> - <suffix domain="gorizia.it" /> - <suffix domain="go.it" /> - <suffix domain="grosseto.it" /> - <suffix domain="gr.it" /> - <suffix domain="imperia.it" /> - <suffix domain="im.it" /> - <suffix domain="isernia.it" /> - <suffix domain="is.it" /> - <suffix domain="laquila.it" /> - <suffix domain="aquila.it" /> - <suffix domain="aq.it" /> - <suffix domain="la-spezia.it" /> - <suffix domain="laspezia.it" /> - <suffix domain="sp.it" /> - <suffix domain="latina.it" /> - <suffix domain="lt.it" /> - <suffix domain="lecce.it" /> - <suffix domain="le.it" /> - <suffix domain="lecco.it" /> - <suffix domain="lc.it" /> - <suffix domain="livorno.it" /> - <suffix domain="li.it" /> - <suffix domain="lodi.it" /> - <suffix domain="lo.it" /> - <suffix domain="lucca.it" /> - <suffix domain="lu.it" /> - <suffix domain="macerata.it" /> - <suffix domain="mc.it" /> - <suffix domain="mantova.it" /> - <suffix domain="mn.it" /> - <suffix domain="massa-carrara.it" /> - <suffix domain="massacarrara.it" /> - <suffix domain="ms.it" /> - <suffix domain="matera.it" /> - <suffix domain="mt.it" /> - <suffix domain="messina.it" /> - <suffix domain="me.it" /> - <suffix domain="milano.it" /> - <suffix domain="milan.it" /> - <suffix domain="mi.it" /> - <suffix domain="modena.it" /> - <suffix domain="mo.it" /> - <suffix domain="monza.it" /> - <suffix domain="napoli.it" /> - <suffix domain="naples.it" /> - <suffix domain="na.it" /> - <suffix domain="novara.it" /> - <suffix domain="no.it" /> - <suffix domain="nuoro.it" /> - <suffix domain="nu.it" /> - <suffix domain="oristano.it" /> - <suffix domain="or.it" /> - <suffix domain="padova.it" /> - <suffix domain="padua.it" /> - <suffix domain="pd.it" /> - <suffix domain="palermo.it" /> - <suffix domain="pa.it" /> - <suffix domain="parma.it" /> - <suffix domain="pr.it" /> - <suffix domain="pavia.it" /> - <suffix domain="pv.it" /> - <suffix domain="perugia.it" /> - <suffix domain="pg.it" /> - <suffix domain="pescara.it" /> - <suffix domain="pe.it" /> - <suffix domain="pesaro-urbino.it" /> - <suffix domain="pesarourbino.it" /> - <suffix domain="pu.it" /> - <suffix domain="piacenza.it" /> - <suffix domain="pc.it" /> - <suffix domain="pisa.it" /> - <suffix domain="pi.it" /> - <suffix domain="pistoia.it" /> - <suffix domain="pt.it" /> - <suffix domain="pordenone.it" /> - <suffix domain="pn.it" /> - <suffix domain="potenza.it" /> - <suffix domain="pz.it" /> - <suffix domain="prato.it" /> - <suffix domain="po.it" /> - <suffix domain="ragusa.it" /> - <suffix domain="rg.it" /> - <suffix domain="ravenna.it" /> - <suffix domain="ra.it" /> - <suffix domain="reggio-calabria.it" /> - <suffix domain="reggiocalabria.it" /> - <suffix domain="rc.it" /> - <suffix domain="reggio-emilia.it" /> - <suffix domain="reggioemilia.it" /> - <suffix domain="re.it" /> - <suffix domain="rieti.it" /> - <suffix domain="ri.it" /> - <suffix domain="rimini.it" /> - <suffix domain="rn.it" /> - <suffix domain="roma.it" /> - <suffix domain="rome.it" /> - <suffix domain="rm.it" /> - <suffix domain="rovigo.it" /> - <suffix domain="ro.it" /> - <suffix domain="salerno.it" /> - <suffix domain="sa.it" /> - <suffix domain="sassari.it" /> - <suffix domain="ss.it" /> - <suffix domain="savona.it" /> - <suffix domain="sv.it" /> - <suffix domain="siena.it" /> - <suffix domain="si.it" /> - <suffix domain="siracusa.it" /> - <suffix domain="sr.it" /> - <suffix domain="sondrio.it" /> - <suffix domain="so.it" /> - <suffix domain="taranto.it" /> - <suffix domain="ta.it" /> - <suffix domain="teramo.it" /> - <suffix domain="te.it" /> - <suffix domain="terni.it" /> - <suffix domain="tr.it" /> - <suffix domain="torino.it" /> - <suffix domain="turin.it" /> - <suffix domain="to.it" /> - <suffix domain="trapani.it" /> - <suffix domain="tp.it" /> - <suffix domain="trento.it" /> - <suffix domain="trentino.it" /> - <suffix domain="tn.it" /> - <suffix domain="treviso.it" /> - <suffix domain="tv.it" /> - <suffix domain="trieste.it" /> - <suffix domain="ts.it" /> - <suffix domain="udine.it" /> - <suffix domain="ud.it" /> - <suffix domain="varese.it" /> - <suffix domain="va.it" /> - <suffix domain="venezia.it" /> - <suffix domain="venice.it" /> - <suffix domain="ve.it" /> - <suffix domain="verbania.it" /> - <suffix domain="vb.it" /> - <suffix domain="vercelli.it" /> - <suffix domain="vc.it" /> - <suffix domain="verona.it" /> - <suffix domain="vr.it" /> - <suffix domain="vibo-valentia.it" /> - <suffix domain="vibovalentia.it" /> - <suffix domain="vv.it" /> - <suffix domain="vicenza.it" /> - <suffix domain="vi.it" /> - <suffix domain="viterbo.it" /> - <suffix domain="vt.it" /> - - <!-- je : http://www.channelisles.net/tandc.shtml--> - <suffix domain="co.je" /> - <suffix domain="org.je" /> - <suffix domain="net.je" /> - <suffix domain="sch.je" /> - <suffix domain="gov.je" /> - - <!-- jm : http://www.com.jm/register.html--> - - <!-- jo : http://www.nis.gov.jo/dns/reg.html--> - <suffix domain="com.jo" /> - <suffix domain="org.jo" /> - <suffix domain="net.jo" /> - <suffix domain="edu.jo" /> - <suffix domain="gov.jo" /> - <suffix domain="mil.jo" /> - <suffix domain="myname.jo" /> - - <!-- jobs : http://en.wikipedia.org/wiki/.jobs--> - - <!-- jp : http://en.wikipedia.org/wiki/.jp--> - <suffix domain="ac.jp" /> - <suffix domain="ad.jp" /> - <suffix domain="co.jp" /> - <suffix domain="ed.jp" /> - <suffix domain="go.jp" /> - <suffix domain="gr.jp" /> - <suffix domain="lg.jp" /> - <suffix domain="ne.jp" /> - <suffix domain="or.jp" /> - <!-- jp geographical names--> - <!-- I can't find an official english explanantrion, but used https://bugzilla.mozilla.org/show_bug.cgi?id=252342#c31--> - <suffix domain="aichi.jp" /> - <suffix domain="akita.jp" /> - <suffix domain="aomori.jp" /> - <suffix domain="chiba.jp" /> - <suffix domain="ehime.jp" /> - <suffix domain="fukui.jp" /> - <suffix domain="fukuoka.jp" /> - <suffix domain="fukushima.jp" /> - <suffix domain="gifu.jp" /> - <suffix domain="gunma.jp" /> - <suffix domain="hiroshima.jp" /> - <suffix domain="hokkaido.jp" /> - <suffix domain="hyogo.jp" /> - <suffix domain="ibaraki.jp" /> - <suffix domain="ishikawa.jp" /> - <suffix domain="iwate.jp" /> - <suffix domain="kagawa.jp" /> - <suffix domain="kagoshima.jp" /> - <suffix domain="kanagawa.jp" /> - <suffix domain="kawasaki.jp" /> - <suffix domain="kitakyushu.jp" /> - <suffix domain="kobe.jp" /> - <suffix domain="kochi.jp" /> - <suffix domain="kumamoto.jp" /> - <suffix domain="kyoto.jp" /> - <suffix domain="mie.jp" /> - <suffix domain="miyagi.jp" /> - <suffix domain="miyazaki.jp" /> - <suffix domain="nagano.jp" /> - <suffix domain="nagasaki.jp" /> - <suffix domain="nagoya.jp" /> - <suffix domain="nara.jp" /> - <suffix domain="niigata.jp" /> - <suffix domain="oita.jp" /> - <suffix domain="okayama.jp" /> - <suffix domain="okinawa.jp" /> - <suffix domain="osaka.jp" /> - <suffix domain="saga.jp" /> - <suffix domain="saitama.jp" /> - <suffix domain="sapporo.jp" /> - <suffix domain="sendai.jp" /> - <suffix domain="shiga.jp" /> - <suffix domain="shimane.jp" /> - <suffix domain="shizuoka.jp" /> - <suffix domain="tochigi.jp" /> - <suffix domain="tokushima.jp" /> - <suffix domain="tokyo.jp" /> - <suffix domain="tottori.jp" /> - <suffix domain="toyama.jp" /> - <suffix domain="wakayama.jp" /> - <suffix domain="yamagata.jp" /> - <suffix domain="yamaguchi.jp" /> - <suffix domain="yamanashi.jp" /> - <suffix domain="yokohama.jp" /> - <suffix domain="metro.tokyo.jp" /> - <suffix domain="pref.aichi.jp" /> - <suffix domain="pref.akita.jp" /> - <suffix domain="pref.aomori.jp" /> - <suffix domain="pref.chiba.jp" /> - <suffix domain="pref.ehime.jp" /> - <suffix domain="pref.fukui.jp" /> - <suffix domain="pref.fukuoka.jp" /> - <suffix domain="pref.fukushima.jp" /> - <suffix domain="pref.gifu.jp" /> - <suffix domain="pref.gunma.jp" /> - <suffix domain="pref.hiroshima.jp" /> - <suffix domain="pref.hokkaido.jp" /> - <suffix domain="pref.hyogo.jp" /> - <suffix domain="pref.ibaraki.jp" /> - <suffix domain="pref.ishikawa.jp" /> - <suffix domain="pref.iwate.jp" /> - <suffix domain="pref.kagawa.jp" /> - <suffix domain="pref.kagoshima.jp" /> - <suffix domain="pref.kanagawa.jp" /> - <suffix domain="pref.kochi.jp" /> - <suffix domain="pref.kumamoto.jp" /> - <suffix domain="pref.kyoto.jp" /> - <suffix domain="pref.mie.jp" /> - <suffix domain="pref.miyagi.jp" /> - <suffix domain="pref.miyazaki.jp" /> - <suffix domain="pref.nagano.jp" /> - <suffix domain="pref.nagasaki.jp" /> - <suffix domain="pref.nara.jp" /> - <suffix domain="pref.niigata.jp" /> - <suffix domain="pref.oita.jp" /> - <suffix domain="pref.okayama.jp" /> - <suffix domain="pref.okinawa.jp" /> - <suffix domain="pref.osaka.jp" /> - <suffix domain="pref.saga.jp" /> - <suffix domain="pref.saitama.jp" /> - <suffix domain="pref.shiga.jp" /> - <suffix domain="pref.shimane.jp" /> - <suffix domain="pref.shizuoka.jp" /> - <suffix domain="pref.tochigi.jp" /> - <suffix domain="pref.tokushima.jp" /> - <suffix domain="pref.tottori.jp" /> - <suffix domain="pref.toyama.jp" /> - <suffix domain="pref.wakayama.jp" /> - <suffix domain="pref.yamagata.jp" /> - <suffix domain="pref.yamaguchi.jp" /> - <suffix domain="pref.yamanashi.jp" /> - <suffix domain="city.chiba.jp" /> - <suffix domain="city.fukuoka.jp" /> - <suffix domain="city.hiroshima.jp" /> - <suffix domain="city.kawasaki.jp" /> - <suffix domain="city.kitakyushu.jp" /> - <suffix domain="city.kobe.jp" /> - <suffix domain="city.kyoto.jp" /> - <suffix domain="city.nagoya.jp" /> - <suffix domain="city.osaka.jp" /> - <suffix domain="city.saitama.jp" /> - <suffix domain="city.sapporo.jp" /> - <suffix domain="city.sendai.jp" /> - <suffix domain="city.shizuoka.jp" /> - <suffix domain="city.yokohama.jp" /> - - <!-- ke : http://www.kenic.or.ke/index.php?option=com_content&task=view&id=117&Itemid=145--> - - <!-- kg : http://www.domain.kg/dmn_n.html--> - <suffix domain="org.kg" /> - <suffix domain="net.kg" /> - <suffix domain="com.kg" /> - <suffix domain="edu.kg" /> - <suffix domain="gov.kg" /> - <suffix domain="mil.kg" /> - - <!-- kh : http://www.mptc.gov.kh/dns_registration.htm--> - - <!-- ki : http://www.ki/dns/index.html--> - <suffix domain="edu.ki" /> - <suffix domain="biz.ki" /> - <suffix domain="net.ki" /> - <suffix domain="org.ki" /> - <suffix domain="gov.ki" /> - <suffix domain="info.ki" /> - <suffix domain="com.ki" /> - - <!-- km : http://en.wikipedia.org/wiki/.km--> - - <!-- kn : http://en.wikipedia.org/wiki/.kn--> - - <!-- kr : http://domain.nida.or.kr/eng/structure.jsp--> - <suffix domain="ac.kr" /> - <suffix domain="co.kr" /> - <suffix domain="go.kr" /> - <suffix domain="ne.kr" /> - <suffix domain="or.kr" /> - <suffix domain="re.kr" /> - <suffix domain="pe.kr" /> - <suffix domain="՜Ꞁ.kr" /> - - <!-- kw : http://en.wikipedia.org/wiki/.kw--> - - <!-- ky : http://www.icta.ky/da_ky_reg_dom.php--> - <suffix domain="edu.ky" /> - <suffix domain="gov.ky" /> - <suffix domain="com.ky" /> - <suffix domain="org.ky" /> - <suffix domain="net.ky" /> - - <!-- kz : http://en.wikipedia.org/wiki/.kz--> - <suffix domain="org.kz" /> - <suffix domain="edu.kz" /> - <suffix domain="net.kz" /> - <suffix domain="gov.kz" /> - <suffix domain="mil.kz" /> - <suffix domain="com.kz" /> - - <!-- la : http://en.wikipedia.org/wiki/.la--> - - <!-- lb : http://en.wikipedia.org/wiki/.lb--> - - <!-- lc : http://en.wikipedia.org/wiki/.lc--> - <suffix domain="com.lc" /> - <suffix domain="org.lc" /> - <suffix domain="edu.lc" /> - <suffix domain="gov.lc" /> - - <!-- li : http://en.wikipedia.org/wiki/.li--> - - <!-- lk : http://www.nic.lk/seclevpr.html--> - <suffix domain="gov.lk" /> - <suffix domain="sch.lk" /> - <suffix domain="net.lk" /> - <suffix domain="int.lk" /> - <suffix domain="com.lk" /> - <suffix domain="org.lk" /> - <suffix domain="edu.lk" /> - <suffix domain="ngo.lk" /> - <suffix domain="soc.lk" /> - <suffix domain="web.lk" /> - <suffix domain="ltd.lk" /> - <suffix domain="assn.lk" /> - <suffix domain="grp.lk" /> - <suffix domain="hotel.lk" /> - - <!-- lr : http://psg.com/dns/lr/lr.txt--> - - <!-- ls : http://en.wikipedia.org/wiki/.ls--> - <suffix domain="co.ls" /> - <suffix domain="org.ls" /> - - <!-- lt : http://en.wikipedia.org/wiki/.lt--> - - <!-- lu : http://www.dns.lu/en/--> - - <!-- lv : http://www.nic.lv/DNS/En/generic.php--> - <suffix domain="com.lv" /> - <suffix domain="edu.lv" /> - <suffix domain="gov.lv" /> - <suffix domain="org.lv" /> - <suffix domain="mil.lv" /> - <suffix domain="id.lv" /> - <suffix domain="net.lv" /> - <suffix domain="asn.lv" /> - <suffix domain="conf.lv" /> - - <!-- ly : http://www.nic.ly/regulations.php--> - <suffix domain="com.ly" /> - <suffix domain="net.ly" /> - <suffix domain="gov.ly" /> - <suffix domain="plc.ly" /> - <suffix domain="edu.ly" /> - <suffix domain="sch.ly" /> - <suffix domain="med.ly" /> - <suffix domain="org.ly" /> - <suffix domain="id.ly" /> - - <!-- ma : http://en.wikipedia.org/wiki/.ma--> - <!-- list of 2nd level tlds ?--> - <suffix domain="co.ma" /> - <suffix domain="net.ma" /> - <suffix domain="gov.ma" /> - <suffix domain="org.ma" /> - - <!-- mc : http://www.nic.mc/--> - <suffix domain="tm.mc" /> - <suffix domain="asso.mc" /> - - <!-- md : http://en.wikipedia.org/wiki/.md--> - - <!-- mg : http://www.nic.mg/tarif.htm--> - <suffix domain="org.mg" /> - <suffix domain="nom.mg" /> - <suffix domain="gov.mg" /> - <suffix domain="prd.mg" /> - <suffix domain="tm.mg" /> - <suffix domain="edu.mg" /> - <suffix domain="mil.mg" /> - <suffix domain="com.mg" /> - - <!-- mh : http://en.wikipedia.org/wiki/.mh--> - - <!-- mil : http://en.wikipedia.org/wiki/.mil--> - - <!-- mk : http://en.wikipedia.org/wiki/.mk--> - <!-- list of 2nd level tlds ?--> - <suffix domain="com.mk" /> - <suffix domain="gov.mk" /> - <suffix domain="org.mk" /> - <suffix domain="net.mk" /> - <suffix domain="edu.mk" /> - - <!-- ml : http://www.gobin.info/domainname/ml-template.doc--> - - <!-- mm : http://en.wikipedia.org/wiki/.mm--> - - <!-- mn : http://en.wikipedia.org/wiki/.mn--> - <suffix domain="gov.mn" /> - <suffix domain="edu.mn" /> - <suffix domain="org.mn" /> - - <!-- mo : http://www.monic.net.mo/--> - <suffix domain="com.mo" /> - <suffix domain="net.mo" /> - <suffix domain="org.mo" /> - <suffix domain="edu.mo" /> - <suffix domain="gov.mo" /> - - <!-- mobi : http://en.wikipedia.org/wiki/.mobi--> - - <!-- mp : http://www.dot.mp/--> - - <!-- mq : http://en.wikipedia.org/wiki/.mq--> - - <!-- mr : http://en.wikipedia.org/wiki/.mr--> - - <!-- ms : http://en.wikipedia.org/wiki/.ms--> - - <!-- mt : https://www.nic.org.mt/dotmt/--> - - <!-- mu : http://en.wikipedia.org/wiki/.mu--> - <!-- list of 2nd level tlds ?--> - - <!-- museum : http://about.museum/naming/--> - <!-- there are 2nd-level TLD's, but there's no list--> - - <!-- mv : http://en.wikipedia.org/wiki/.mv--> - - <!-- mw : http://www.registrar.mw/--> - <suffix domain="ac.mw" /> - <suffix domain="biz.mw" /> - <suffix domain="co.mw" /> - <suffix domain="com.mw" /> - <suffix domain="coop.mw" /> - <suffix domain="edu.mw" /> - <suffix domain="gov.mw" /> - <suffix domain="int.mw" /> - <suffix domain="net.mw" /> - <suffix domain="org.mw" /> - - <!-- mx : http://www.nic.mx/--> - - <!-- my : http://www.mynic.net.my/--> - - <!-- mz : http://www.gobin.info/domainname/mz-template.doc--> - - <!-- na : http://www.na-nic.com.na/--> - <!-- list of 2nd level tlds ?--> - - <!-- name : has 2nd-level tlds, but there's no list of them--> - - <!-- nc : http://www.cctld.nc/--> - - <!-- ne : http://en.wikipedia.org/wiki/.ne--> - - <!-- net : http://en.wikipedia.org/wiki/.net--> - - <!-- nf : http://en.wikipedia.org/wiki/.nf--> - <suffix domain="com.nf" /> - <suffix domain="net.nf" /> - <suffix domain="per.nf" /> - <suffix domain="rec.nf" /> - <suffix domain="web.nf" /> - <suffix domain="arts.nf" /> - <suffix domain="firm.nf" /> - <suffix domain="info.nf" /> - <suffix domain="other.nf" /> - <suffix domain="store.nf" /> - - <!-- ng : http://psg.com/dns/ng/--> - - <!-- ni : http://www.nic.ni/dominios.htm--> - - <!-- nl : http://www.domain-registry.nl/ace.php/c,728,122,,,,Home.html--> - - <!-- no : http://www.norid.no/regelverk/index.en.html--> - <suffix domain="fhs.no" /> - <suffix domain="vgs.no" /> - <suffix domain="fylkesbibl.no" /> - <suffix domain="folkebibl.no" /> - <suffix domain="museum.no" /> - <suffix domain="idrett.no" /> - <suffix domain="mil.no" /> - <suffix domain="stat.no" /> - <suffix domain="dep.no" /> - <suffix domain="kommune.no" /> - <suffix domain="herad.no" /> - <suffix domain="priv.no" /> - <!-- no geographical names : http://www.norid.no/regelverk/vedlegg-b.en.html--> - <!-- counties--> - <suffix domain="aa.no" /> - <suffix domain="ah.no" /> - <suffix domain="bu.no" /> - <suffix domain="fm.no" /> - <suffix domain="hl.no" /> - <suffix domain="hm.no" /> - <suffix domain="jan-mayen.no" /> - <suffix domain="mr.no" /> - <suffix domain="nl.no" /> - <suffix domain="nt.no" /> - <suffix domain="of.no" /> - <suffix domain="ol.no" /> - <suffix domain="oslo.no" /> - <suffix domain="rl.no" /> - <suffix domain="sf.no" /> - <suffix domain="st.no" /> - <suffix domain="svalbard.no" /> - <suffix domain="tm.no" /> - <suffix domain="tr.no" /> - <suffix domain="va.no" /> - <suffix domain="vf.no" /> - <!-- primary and lower secondary schools per county--> - <suffix domain="gs.aa.no" /> - <suffix domain="gs.ah.no" /> - <suffix domain="gs.bu.no" /> - <suffix domain="gs.fm.no" /> - <suffix domain="gs.hl.no" /> - <suffix domain="gs.hm.no" /> - <suffix domain="gs.jan-mayen.no" /> - <suffix domain="gs.mr.no" /> - <suffix domain="gs.nl.no" /> - <suffix domain="gs.nt.no" /> - <suffix domain="gs.of.no" /> - <suffix domain="gs.ol.no" /> - <suffix domain="gs.oslo.no" /> - <suffix domain="gs.rl.no" /> - <suffix domain="gs.sf.no" /> - <suffix domain="gs.st.no" /> - <suffix domain="gs.svalbard.no" /> - <suffix domain="gs.tm.no" /> - <suffix domain="gs.tr.no" /> - <suffix domain="gs.va.no" /> - <suffix domain="gs.vf.no" /> - <!-- cities--> - <suffix domain="akrehamn.no" /> - <suffix domain="Ã¥krehamn.no" /> - <suffix domain="algard.no" /> - <suffix domain="Ã¥lgÃ¥rd.no" /> - <suffix domain="arna.no" /> - <suffix domain="brumunddal.no" /> - <suffix domain="bryne.no" /> - <suffix domain="bronnoysund.no" /> - <suffix domain="brÞnnÞysund.no" /> - <suffix domain="drobak.no" /> - <suffix domain="drÞbak.no" /> - <suffix domain="egersund.no" /> - <suffix domain="fetsund.no" /> - <suffix domain="floro.no" /> - <suffix domain="florÞ.no" /> - <suffix domain="fredrikstad.no" /> - <suffix domain="hokksund.no" /> - <suffix domain="honefoss.no" /> - <suffix domain="hÞnefoss.no" /> - <suffix domain="jessheim.no" /> - <suffix domain="jorpeland.no" /> - <suffix domain="jÞrpeland.no" /> - <suffix domain="kirkenes.no" /> - <suffix domain="kopervik.no" /> - <suffix domain="krokstadelva.no" /> - <suffix domain="langevag.no" /> - <suffix domain="langevÃ¥g.no" /> - <suffix domain="leirvik.no" /> - <suffix domain="mjondalen.no" /> - <suffix domain="mjÞndalen.no" /> - <suffix domain="mo-i-rana.no" /> - <suffix domain="mosjoen.no" /> - <suffix domain="mosjÞen.no" /> - <suffix domain="nesoddtangen.no" /> - <suffix domain="orkanger.no" /> - <suffix domain="osoyro.no" /> - <suffix domain="osÞyro.no" /> - <suffix domain="raholt.no" /> - <suffix domain="rÃ¥holt.no" /> - <suffix domain="sandnessjoen.no" /> - <suffix domain="sandnessjÞen.no" /> - <suffix domain="skedsmokorset.no" /> - <suffix domain="slattum.no" /> - <suffix domain="spjelkavik.no" /> - <suffix domain="stathelle.no" /> - <suffix domain="stavern.no" /> - <suffix domain="stjordalshalsen.no" /> - <suffix domain="stjÞrdalshalsen.no" /> - <suffix domain="tananger.no" /> - <suffix domain="tranby.no" /> - <suffix domain="vossevangen.no" /> - <!-- communities--> - <suffix domain="afjord.no" /> - <suffix domain="Ã¥fjord.no" /> - <suffix domain="agdenes.no" /> - <suffix domain="al.no" /> - <suffix domain="Ã¥l.no" /> - <suffix domain="alesund.no" /> - <suffix domain="Ã¥lesund.no" /> - <suffix domain="alstahaug.no" /> - <suffix domain="alta.no" /> - <suffix domain="áltá.no" /> - <suffix domain="alaheadju.no" /> - <suffix domain="álaheadju.no" /> - <suffix domain="alvdal.no" /> - <suffix domain="amli.no" /> - <suffix domain="Ã¥mli.no" /> - <suffix domain="amot.no" /> - <suffix domain="Ã¥mot.no" /> - <suffix domain="andebu.no" /> - <suffix domain="andoy.no" /> - <suffix domain="andÞy.no" /> - <suffix domain="andasuolo.no" /> - <suffix domain="ardal.no" /> - <suffix domain="Ã¥rdal.no" /> - <suffix domain="aremark.no" /> - <suffix domain="arendal.no" /> - <suffix domain="Ã¥s.no" /> - <suffix domain="aseral.no" /> - <suffix domain="Ã¥seral.no" /> - <suffix domain="asker.no" /> - <suffix domain="askim.no" /> - <suffix domain="askvoll.no" /> - <suffix domain="askoy.no" /> - <suffix domain="askÞy.no" /> - <suffix domain="asnes.no" /> - <suffix domain="Ã¥snes.no" /> - <suffix domain="audnedaln.no" /> - <suffix domain="aukra.no" /> - <suffix domain="aure.no" /> - <suffix domain="aurland.no" /> - <suffix domain="aurskog-holand.no" /> - <suffix domain="aurskog-hÞland.no" /> - <suffix domain="austevoll.no" /> - <suffix domain="austrheim.no" /> - <suffix domain="averoy.no" /> - <suffix domain="averÞy.no" /> - <suffix domain="balestrand.no" /> - <suffix domain="ballangen.no" /> - <suffix domain="balat.no" /> - <suffix domain="bálát.no" /> - <suffix domain="balsfjord.no" /> - <suffix domain="bahccavuotna.no" /> - <suffix domain="báhccavuotna.no" /> - <suffix domain="bamble.no" /> - <suffix domain="bardu.no" /> - <suffix domain="beardu.no" /> - <suffix domain="beiarn.no" /> - <suffix domain="bajddar.no" /> - <suffix domain="bájddar.no" /> - <suffix domain="baidar.no" /> - <suffix domain="báidár.no" /> - <suffix domain="berg.no" /> - <suffix domain="bergen.no" /> - <suffix domain="berlevag.no" /> - <suffix domain="berlevÃ¥g.no" /> - <suffix domain="bearalvahki.no" /> - <suffix domain="bearalváhki.no" /> - <suffix domain="bindal.no" /> - <suffix domain="birkenes.no" /> - <suffix domain="bjarkoy.no" /> - <suffix domain="bjarkÞy.no" /> - <suffix domain="bjerkreim.no" /> - <suffix domain="bjugn.no" /> - <suffix domain="bodo.no" /> - <suffix domain="bodÞ.no" /> - <suffix domain="badaddja.no" /> - <suffix domain="bÃ¥dÃ¥ddjÃ¥.no" /> - <suffix domain="budejju.no" /> - <suffix domain="bokn.no" /> - <suffix domain="bremanger.no" /> - <suffix domain="bronnoy.no" /> - <suffix domain="brÞnnÞy.no" /> - <suffix domain="bygland.no" /> - <suffix domain="bykle.no" /> - <suffix domain="barum.no" /> - <suffix domain="bÊrum.no" /> - <suffix domain="bo.telemark.no" /> - <suffix domain="bÞ.telemark.no" /> - <suffix domain="bo.nordland.no" /> - <suffix domain="bÞ.nordland.no" /> - <suffix domain="bievat.no" /> - <suffix domain="bievát.no" /> - <suffix domain="bomlo.no" /> - <suffix domain="bÞmlo.no" /> - <suffix domain="batsfjord.no" /> - <suffix domain="bÃ¥tsfjord.no" /> - <suffix domain="bahcavuotna.no" /> - <suffix domain="báhcavuotna.no" /> - <suffix domain="dovre.no" /> - <suffix domain="drammen.no" /> - <suffix domain="drangedal.no" /> - <suffix domain="dyroy.no" /> - <suffix domain="dyrÞy.no" /> - <suffix domain="donna.no" /> - <suffix domain="dÞnna.no" /> - <suffix domain="eid.no" /> - <suffix domain="eidfjord.no" /> - <suffix domain="eidsberg.no" /> - <suffix domain="eidskog.no" /> - <suffix domain="eidsvoll.no" /> - <suffix domain="eigersund.no" /> - <suffix domain="elverum.no" /> - <suffix domain="enebakk.no" /> - <suffix domain="engerdal.no" /> - <suffix domain="etne.no" /> - <suffix domain="etnedal.no" /> - <suffix domain="evenes.no" /> - <suffix domain="evenassi.no" /> - <suffix domain="evenášši.no" /> - <suffix domain="evje-og-hornnes.no" /> - <suffix domain="farsund.no" /> - <suffix domain="fauske.no" /> - <suffix domain="fuossko.no" /> - <suffix domain="fuoisku.no" /> - <suffix domain="fedje.no" /> - <suffix domain="fet.no" /> - <suffix domain="finnoy.no" /> - <suffix domain="finnÞy.no" /> - <suffix domain="fitjar.no" /> - <suffix domain="fjaler.no" /> - <suffix domain="fjell.no" /> - <suffix domain="flakstad.no" /> - <suffix domain="flatanger.no" /> - <suffix domain="flekkefjord.no" /> - <suffix domain="flesberg.no" /> - <suffix domain="flora.no" /> - <suffix domain="fla.no" /> - <suffix domain="flÃ¥.no" /> - <suffix domain="folldal.no" /> - <suffix domain="forsand.no" /> - <suffix domain="fosnes.no" /> - <suffix domain="fredrikstad.no" /> - <suffix domain="frei.no" /> - <suffix domain="frogn.no" /> - <suffix domain="froland.no" /> - <suffix domain="frosta.no" /> - <suffix domain="frana.no" /> - <suffix domain="frÊna.no" /> - <suffix domain="froya.no" /> - <suffix domain="frÞya.no" /> - <suffix domain="fusa.no" /> - <suffix domain="fyresdal.no" /> - <suffix domain="forde.no" /> - <suffix domain="fÞrde.no" /> - <suffix domain="gamvik.no" /> - <suffix domain="gangaviika.no" /> - <suffix domain="gaular.no" /> - <suffix domain="gausdal.no" /> - <suffix domain="gildeskal.no" /> - <suffix domain="gildeskÃ¥l.no" /> - <suffix domain="giske.no" /> - <suffix domain="gjemnes.no" /> - <suffix domain="gjerdrum.no" /> - <suffix domain="gjerstad.no" /> - <suffix domain="gjesdal.no" /> - <suffix domain="gjovik.no" /> - <suffix domain="gjÞvik.no" /> - <suffix domain="gloppen.no" /> - <suffix domain="gol.no" /> - <suffix domain="gran.no" /> - <suffix domain="grane.no" /> - <suffix domain="granvin.no" /> - <suffix domain="gratangen.no" /> - <suffix domain="grimstad.no" /> - <suffix domain="grong.no" /> - <suffix domain="kraanghke.no" /> - <suffix domain="krÃ¥anghke.no" /> - <suffix domain="grue.no" /> - <suffix domain="gulen.no" /> - <suffix domain="hadsel.no" /> - <suffix domain="halden.no" /> - <suffix domain="halsa.no" /> - <suffix domain="hamar.no" /> - <suffix domain="hamaroy.no" /> - <suffix domain="habmer.no" /> - <suffix domain="hábmer.no" /> - <suffix domain="hapmir.no" /> - <suffix domain="hápmir.no" /> - <suffix domain="hammerfest.no" /> - <suffix domain="hammarfeasta.no" /> - <suffix domain="hámmárfeasta.no" /> - <suffix domain="haram.no" /> - <suffix domain="hareid.no" /> - <suffix domain="harstad.no" /> - <suffix domain="hasvik.no" /> - <suffix domain="aknoluokta.no" /> - <suffix domain="ákŋoluokta.no" /> - <suffix domain="hattfjelldal.no" /> - <suffix domain="aarborte.no" /> - <suffix domain="haugesund.no" /> - <suffix domain="hemne.no" /> - <suffix domain="hemnes.no" /> - <suffix domain="hemsedal.no" /> - <suffix domain="heroy.more-og-romsdal.no" /> - <suffix domain="herÞy.mÞre-og-romsdal.no" /> - <suffix domain="heroy.nordland.no" /> - <suffix domain="herÞy.nordland.no" /> - <suffix domain="hitra.no" /> - <suffix domain="hjartdal.no" /> - <suffix domain="hjelmeland.no" /> - <suffix domain="hobol.no" /> - <suffix domain="hobÞl.no" /> - <suffix domain="hof.no" /> - <suffix domain="hol.no" /> - <suffix domain="hole.no" /> - <suffix domain="holmestrand.no" /> - <suffix domain="holtalen.no" /> - <suffix domain="holtÃ¥len.no" /> - <suffix domain="hornindal.no" /> - <suffix domain="horten.no" /> - <suffix domain="hurdal.no" /> - <suffix domain="hurum.no" /> - <suffix domain="hvaler.no" /> - <suffix domain="hyllestad.no" /> - <suffix domain="hagebostad.no" /> - <suffix domain="hÊgebostad.no" /> - <suffix domain="hoyanger.no" /> - <suffix domain="hÞyanger.no" /> - <suffix domain="hoylandet.no" /> - <suffix domain="hÞylandet.no" /> - <suffix domain="ha.no" /> - <suffix domain="hÃ¥.no" /> - <suffix domain="ibestad.no" /> - <suffix domain="inderoy.no" /> - <suffix domain="inderÞy.no" /> - <suffix domain="iveland.no" /> - <suffix domain="jevnaker.no" /> - <suffix domain="jondal.no" /> - <suffix domain="jolster.no" /> - <suffix domain="jÞlster.no" /> - <suffix domain="karasjok.no" /> - <suffix domain="karasjohka.no" /> - <suffix domain="kárášjohka.no" /> - <suffix domain="karlsoy.no" /> - <suffix domain="galsa.no" /> - <suffix domain="gálsá.no" /> - <suffix domain="karmoy.no" /> - <suffix domain="karmÞy.no" /> - <suffix domain="kautokeino.no" /> - <suffix domain="guovdageaidnu.no" /> - <suffix domain="klepp.no" /> - <suffix domain="klabu.no" /> - <suffix domain="klÊbu.no" /> - <suffix domain="kongsberg.no" /> - <suffix domain="kongsvinger.no" /> - <suffix domain="kragero.no" /> - <suffix domain="kragerÞ.no" /> - <suffix domain="kristiansand.no" /> - <suffix domain="kristiansund.no" /> - <suffix domain="krodsherad.no" /> - <suffix domain="krÞdsherad.no" /> - <suffix domain="kvalsund.no" /> - <suffix domain="rahkkeravju.no" /> - <suffix domain="ráhkkerávju.no" /> - <suffix domain="kvam.no" /> - <suffix domain="kvinesdal.no" /> - <suffix domain="kvinnherad.no" /> - <suffix domain="kviteseid.no" /> - <suffix domain="kvitsoy.no" /> - <suffix domain="kvitsÞy.no" /> - <suffix domain="kvafjord.no" /> - <suffix domain="kvÊfjord.no" /> - <suffix domain="giehtavuoatna.no" /> - <suffix domain="kvanangen.no" /> - <suffix domain="kvÊnangen.no" /> - <suffix domain="navuotna.no" /> - <suffix domain="návuotna.no" /> - <suffix domain="kafjord.no" /> - <suffix domain="kÃ¥fjord.no" /> - <suffix domain="gaivuotna.no" /> - <suffix domain="gáivuotna.no" /> - <suffix domain="larvik.no" /> - <suffix domain="lavangen.no" /> - <suffix domain="lavagis.no" /> - <suffix domain="loabat.no" /> - <suffix domain="loabát.no" /> - <suffix domain="lebesby.no" /> - <suffix domain="davvesiida.no" /> - <suffix domain="leikanger.no" /> - <suffix domain="leirfjord.no" /> - <suffix domain="leka.no" /> - <suffix domain="leksvik.no" /> - <suffix domain="lenvik.no" /> - <suffix domain="leangaviika.no" /> - <suffix domain="lea?gaviika.no" /> - <suffix domain="lesja.no" /> - <suffix domain="levanger.no" /> - <suffix domain="lier.no" /> - <suffix domain="lierne.no" /> - <suffix domain="lillehammer.no" /> - <suffix domain="lillesand.no" /> - <suffix domain="lindesnes.no" /> - <suffix domain="lindas.no" /> - <suffix domain="lindÃ¥s.no" /> - <suffix domain="lom.no" /> - <suffix domain="loppa.no" /> - <suffix domain="lahppi.no" /> - <suffix domain="láhppi.no" /> - <suffix domain="lund.no" /> - <suffix domain="lunner.no" /> - <suffix domain="luroy.no" /> - <suffix domain="lurÞy.no" /> - <suffix domain="luster.no" /> - <suffix domain="lyngdal.no" /> - <suffix domain="lyngen.no" /> - <suffix domain="ivgu.no" /> - <suffix domain="lardal.no" /> - <suffix domain="lerdal.no" /> - <suffix domain="lÊrdal.no" /> - <suffix domain="lodingen.no" /> - <suffix domain="lÞdingen.no" /> - <suffix domain="lorenskog.no" /> - <suffix domain="lÞrenskog.no" /> - <suffix domain="loten.no" /> - <suffix domain="lÞten.no" /> - <suffix domain="malvik.no" /> - <suffix domain="masoy.no" /> - <suffix domain="mÃ¥sÞy.no" /> - <suffix domain="muosat.no" /> - <suffix domain="muosát.no" /> - <suffix domain="mandal.no" /> - <suffix domain="marker.no" /> - <suffix domain="marnardal.no" /> - <suffix domain="masfjorden.no" /> - <suffix domain="meland.no" /> - <suffix domain="meldal.no" /> - <suffix domain="melhus.no" /> - <suffix domain="meloy.no" /> - <suffix domain="melÞy.no" /> - <suffix domain="meraker.no" /> - <suffix domain="merÃ¥ker.no" /> - <suffix domain="moareke.no" /> - <suffix domain="moÃ¥reke.no" /> - <suffix domain="midsund.no" /> - <suffix domain="midtre-gauldal.no" /> - <suffix domain="modalen.no" /> - <suffix domain="modum.no" /> - <suffix domain="molde.no" /> - <suffix domain="moskenes.no" /> - <suffix domain="moss.no" /> - <suffix domain="mosvik.no" /> - <suffix domain="malselv.no" /> - <suffix domain="mÃ¥lselv.no" /> - <suffix domain="malatvuopmi.no" /> - <suffix domain="málatvuopmi.no" /> - <suffix domain="namdalseid.no" /> - <suffix domain="aejrie.no" /> - <suffix domain="namsos.no" /> - <suffix domain="namsskogan.no" /> - <suffix domain="naamesjevuemie.no" /> - <suffix domain="nååmesjevuemie.no" /> - <suffix domain="laakesvuemie.no" /> - <suffix domain="nannestad.no" /> - <suffix domain="narvik.no" /> - <suffix domain="narviika.no" /> - <suffix domain="naustdal.no" /> - <suffix domain="nedre-eiker.no" /> - <suffix domain="nes.akershus.no" /> - <suffix domain="nes.buskerud.no" /> - <suffix domain="nesna.no" /> - <suffix domain="nesodden.no" /> - <suffix domain="nesseby.no" /> - <suffix domain="unjarga.no" /> - <suffix domain="unjárga.no" /> - <suffix domain="nesset.no" /> - <suffix domain="nissedal.no" /> - <suffix domain="nittedal.no" /> - <suffix domain="nord-aurdal.no" /> - <suffix domain="nord-fron.no" /> - <suffix domain="nord-odal.no" /> - <suffix domain="norddal.no" /> - <suffix domain="nordkapp.no" /> - <suffix domain="davvenjarga.no" /> - <suffix domain="davvenjárga.no" /> - <suffix domain="nordre-land.no" /> - <suffix domain="nordreisa.no" /> - <suffix domain="raisa.no" /> - <suffix domain="ráisa.no" /> - <suffix domain="nore-og-uvdal.no" /> - <suffix domain="notodden.no" /> - <suffix domain="naroy.no" /> - <suffix domain="nÊrÞy.no" /> - <suffix domain="notteroy.no" /> - <suffix domain="nÞtterÞy.no" /> - <suffix domain="odda.no" /> - <suffix domain="oksnes.no" /> - <suffix domain="`Þksnes.no" /> - <suffix domain="oppdal.no" /> - <suffix domain="oppegard.no" /> - <suffix domain="oppegÃ¥rd.no" /> - <suffix domain="orkdal.no" /> - <suffix domain="orland.no" /> - <suffix domain="Þrland.no" /> - <suffix domain="orskog.no" /> - <suffix domain="Þrskog.no" /> - <suffix domain="orsta.no" /> - <suffix domain="Þrsta.no" /> - <suffix domain="os.hedmark.no" /> - <suffix domain="os.hordaland.no" /> - <suffix domain="osen.no" /> - <suffix domain="osteroy.no" /> - <suffix domain="osterÞy.no" /> - <suffix domain="ostre-toten.no" /> - <suffix domain="Þstre-toten.no" /> - <suffix domain="overhalla.no" /> - <suffix domain="ovre-eiker.no" /> - <suffix domain="Þvre-eiker.no" /> - <suffix domain="oyer.no" /> - <suffix domain="Þyer.no" /> - <suffix domain="oygarden.no" /> - <suffix domain="Þygarden.no" /> - <suffix domain="oystre-slidre.no" /> - <suffix domain="Þystre-slidre.no" /> - <suffix domain="porsanger.no" /> - <suffix domain="porsangu.no" /> - <suffix domain="porsáŋgu.no" /> - <suffix domain="porsgrunn.no" /> - <suffix domain="radoy.no" /> - <suffix domain="radÞy.no" /> - <suffix domain="rakkestad.no" /> - <suffix domain="rana.no" /> - <suffix domain="ruovat.no" /> - <suffix domain="randaberg.no" /> - <suffix domain="rauma.no" /> - <suffix domain="rendalen.no" /> - <suffix domain="rennebu.no" /> - <suffix domain="rennesoy.no" /> - <suffix domain="rennesÞy.no" /> - <suffix domain="rindal.no" /> - <suffix domain="ringebu.no" /> - <suffix domain="ringerike.no" /> - <suffix domain="ringsaker.no" /> - <suffix domain="rissa.no" /> - <suffix domain="risor.no" /> - <suffix domain="risÞr.no" /> - <suffix domain="roan.no" /> - <suffix domain="rollag.no" /> - <suffix domain="rygge.no" /> - <suffix domain="ralingen.no" /> - <suffix domain="rÊlingen.no" /> - <suffix domain="rodoy.no" /> - <suffix domain="rÞdÞy.no" /> - <suffix domain="romskog.no" /> - <suffix domain="rÞmskog.no" /> - <suffix domain="roros.no" /> - <suffix domain="rÞros.no" /> - <suffix domain="rost.no" /> - <suffix domain="rÞst.no" /> - <suffix domain="royken.no" /> - <suffix domain="rÞyken.no" /> - <suffix domain="royrvik.no" /> - <suffix domain="rÞyrvik.no" /> - <suffix domain="rade.no" /> - <suffix domain="rÃ¥de.no" /> - <suffix domain="salangen.no" /> - <suffix domain="siellak.no" /> - <suffix domain="saltdal.no" /> - <suffix domain="sálát.no" /> - <suffix domain="sálat.no" /> - <suffix domain="samnanger.no" /> - <suffix domain="sande.more-og-romsdal.no" /> - <suffix domain="sande.mÞre-og-romsdal.no" /> - <suffix domain="sande.vestfold.no" /> - <suffix domain="sandefjord.no" /> - <suffix domain="sandnes.no" /> - <suffix domain="sandoy.no" /> - <suffix domain="sandÞy.no" /> - <suffix domain="sarpsborg.no" /> - <suffix domain="sauda.no" /> - <suffix domain="sauherad.no" /> - <suffix domain="sel.no" /> - <suffix domain="selbu.no" /> - <suffix domain="selje.no" /> - <suffix domain="seljord.no" /> - <suffix domain="sigdal.no" /> - <suffix domain="siljan.no" /> - <suffix domain="sirdal.no" /> - <suffix domain="skaun.no" /> - <suffix domain="skedsmo.no" /> - <suffix domain="ski.no" /> - <suffix domain="skien.no" /> - <suffix domain="skiptvet.no" /> - <suffix domain="skjervoy.no" /> - <suffix domain="skjervÞy.no" /> - <suffix domain="skierva.no" /> - <suffix domain="skiervá.no" /> - <suffix domain="skjak.no" /> - <suffix domain="skjÃ¥k.no" /> - <suffix domain="skodje.no" /> - <suffix domain="skanland.no" /> - <suffix domain="skÃ¥nland.no" /> - <suffix domain="skanit.no" /> - <suffix domain="skánit.no" /> - <suffix domain="smola.no" /> - <suffix domain="smÞla.no" /> - <suffix domain="snillfjord.no" /> - <suffix domain="snasa.no" /> - <suffix domain="snÃ¥sa.no" /> - <suffix domain="snoasa.no" /> - <suffix domain="snaase.no" /> - <suffix domain="snÃ¥ase.no" /> - <suffix domain="sogndal.no" /> - <suffix domain="sokndal.no" /> - <suffix domain="sola.no" /> - <suffix domain="solund.no" /> - <suffix domain="songdalen.no" /> - <suffix domain="sortland.no" /> - <suffix domain="spydeberg.no" /> - <suffix domain="stange.no" /> - <suffix domain="stavanger.no" /> - <suffix domain="steigen.no" /> - <suffix domain="steinkjer.no" /> - <suffix domain="stjordal.no" /> - <suffix domain="stjÞrdal.no" /> - <suffix domain="stokke.no" /> - <suffix domain="stor-elvdal.no" /> - <suffix domain="stord.no" /> - <suffix domain="stordal.no" /> - <suffix domain="storfjord.no" /> - <suffix domain="omasvuotna.no" /> - <suffix domain="strand.no" /> - <suffix domain="stranda.no" /> - <suffix domain="stryn.no" /> - <suffix domain="sula.no" /> - <suffix domain="suldal.no" /> - <suffix domain="sund.no" /> - <suffix domain="sunndal.no" /> - <suffix domain="surnadal.no" /> - <suffix domain="sveio.no" /> - <suffix domain="svelvik.no" /> - <suffix domain="sykkylven.no" /> - <suffix domain="sogne.no" /> - <suffix domain="sÞgne.no" /> - <suffix domain="somna.no" /> - <suffix domain="sÞmna.no" /> - <suffix domain="sondre-land.no" /> - <suffix domain="sÞndre-land.no" /> - <suffix domain="sor-aurdal.no" /> - <suffix domain="sÞr-aurdal.no" /> - <suffix domain="sor-fron.no" /> - <suffix domain="sÞr-fron.no" /> - <suffix domain="sor-odal.no" /> - <suffix domain="sÞr-odal.no" /> - <suffix domain="sor-varanger.no" /> - <suffix domain="sÞr-varanger.no" /> - <suffix domain="matta-varjjat.no" /> - <suffix domain="mátta-várjjat.no" /> - <suffix domain="sorfold.no" /> - <suffix domain="sÞrfold.no" /> - <suffix domain="sorreisa.no" /> - <suffix domain="sÞrreisa.no" /> - <suffix domain="sorum.no" /> - <suffix domain="sÞrum.no" /> - <suffix domain="tana.no" /> - <suffix domain="deatnu.no" /> - <suffix domain="time.no" /> - <suffix domain="tingvoll.no" /> - <suffix domain="tinn.no" /> - <suffix domain="tjeldsund.no" /> - <suffix domain="dielddanuorri.no" /> - <suffix domain="tjome.no" /> - <suffix domain="tjÞme.no" /> - <suffix domain="tokke.no" /> - <suffix domain="tolga.no" /> - <suffix domain="torsken.no" /> - <suffix domain="tranoy.no" /> - <suffix domain="tranÞy.no" /> - <suffix domain="tromso.no" /> - <suffix domain="tromsÞ.no" /> - <suffix domain="tromsa.no" /> - <suffix domain="romsa.no" /> - <suffix domain="trondheim.no" /> - <suffix domain="troandin.no" /> - <suffix domain="trysil.no" /> - <suffix domain="trana.no" /> - <suffix domain="trÊna.no" /> - <suffix domain="trogstad.no" /> - <suffix domain="trÞgstad.no" /> - <suffix domain="tvedestrand.no" /> - <suffix domain="tydal.no" /> - <suffix domain="tynset.no" /> - <suffix domain="tysfjord.no" /> - <suffix domain="divtasvuodna.no" /> - <suffix domain="divttasvuotna.no" /> - <suffix domain="tysnes.no" /> - <suffix domain="tysvar.no" /> - <suffix domain="tysvÊr.no" /> - <suffix domain="tonsberg.no" /> - <suffix domain="tÞnsberg.no" /> - <suffix domain="ullensaker.no" /> - <suffix domain="ullensvang.no" /> - <suffix domain="ulvik.no" /> - <suffix domain="utsira.no" /> - <suffix domain="vadso.no" /> - <suffix domain="vadsÞ.no" /> - <suffix domain="cahcesuolo.no" /> - <suffix domain="cáhcesuolo.no" /> - <suffix domain="vaksdal.no" /> - <suffix domain="valle.no" /> - <suffix domain="vang.no" /> - <suffix domain="vanylven.no" /> - <suffix domain="vardo.no" /> - <suffix domain="vardÞ.no" /> - <suffix domain="varggat.no" /> - <suffix domain="várggát.no" /> - <suffix domain="vefsn.no" /> - <suffix domain="vaapste.no" /> - <suffix domain="vega.no" /> - <suffix domain="vegarshei.no" /> - <suffix domain="vegÃ¥rshei.no" /> - <suffix domain="vennesla.no" /> - <suffix domain="verdal.no" /> - <suffix domain="verran.no" /> - <suffix domain="vestby.no" /> - <suffix domain="vestnes.no" /> - <suffix domain="vestre-slidre.no" /> - <suffix domain="vestre-toten.no" /> - <suffix domain="vestvagoy.no" /> - <suffix domain="vestvÃ¥gÞy.no" /> - <suffix domain="vevelstad.no" /> - <suffix domain="vik.no" /> - <suffix domain="vikna.no" /> - <suffix domain="vindafjord.no" /> - <suffix domain="volda.no" /> - <suffix domain="voss.no" /> - <suffix domain="varoy.no" /> - <suffix domain="vÊrÞy.no" /> - <suffix domain="vagan.no" /> - <suffix domain="vÃ¥gan.no" /> - <suffix domain="voagat.no" /> - <suffix domain="vagsoy.no" /> - <suffix domain="vÃ¥gsÞy.no" /> - <suffix domain="vaga.no" /> - <suffix domain="vÃ¥gÃ¥.no" /> - <suffix domain="valer.ostfold.no" /> - <suffix domain="vÃ¥ler.Þstfold.no" /> - <suffix domain="valer.hedmark.no" /> - <suffix domain="vÃ¥ler.hedmark.no" /> - - <!-- np : http://www.mos.com.np/register.html--> - - <!-- nr : http://cenpac.net.nr/dns/index.html--> - <suffix domain="biz.nr" /> - <suffix domain="info.nr" /> - <suffix domain="gov.nr" /> - <suffix domain="edu.nr" /> - <suffix domain="org.nr" /> - <suffix domain="net.nr" /> - <suffix domain="com.nr" /> - - <!-- nu : http://en.wikipedia.org/wiki/.nu--> - - <!-- nz : http://en.wikipedia.org/wiki/.nz--> - - <!-- om : http://en.wikipedia.org/wiki/.om--> - - <!-- org : http://en.wikipedia.org/wiki/.og--> - - <!-- pa : http://www.nic.pa/--> - - <!-- pe : http://www.nic.pe/normas-proced-i.htm--> - - <!-- pf : http://www.gobin.info/domainname/formulaire-pf.pdf--> - <suffix domain="com.pf" /> - <suffix domain="org.pf" /> - <suffix domain="edu.pf" /> - - <!-- pg : http://en.wikipedia.org/wiki/.pg--> - - <!-- ph : http://www.domains.ph/FAQ2.asp--> - <!-- list of 2nd level tlds ?--> - <suffix domain="com.ph" /> - <suffix domain="net.ph" /> - <suffix domain="org.ph" /> - <suffix domain="gov.ph" /> - <suffix domain="edu.ph" /> - <suffix domain="ngo.ph" /> - <suffix domain="mil.ph" /> - - <!-- pk : http://pk5.pknic.net.pk/pk5/msgNamepk.PK--> - <suffix domain="com.pk" /> - <suffix domain="net.pk" /> - <suffix domain="edu.pk" /> - <suffix domain="org.pk" /> - <suffix domain="fam.pk" /> - <suffix domain="biz.pk" /> - <suffix domain="web.pk" /> - <suffix domain="gov.pk" /> - <suffix domain="gob.pk" /> - <suffix domain="gok.pk" /> - <suffix domain="gon.pk" /> - <suffix domain="gop.pk" /> - <suffix domain="gos.pk" /> - <suffix domain="goa.pk" /> - <suffix domain="info.pk" /> - - <!-- pl : http://www.dns.pl/english/--> - <!-- NASK functional domains (nask.pl / dns.pl) : http://www.dns.pl/english/dns-funk.html--> - <suffix domain="aid.pl" /> - <suffix domain="agro.pl" /> - <suffix domain="atm.pl" /> - <suffix domain="auto.pl" /> - <suffix domain="biz.pl" /> - <suffix domain="com.pl" /> - <suffix domain="edu.pl" /> - <suffix domain="gmina.pl" /> - <suffix domain="gsm.pl" /> - <suffix domain="info.pl" /> - <suffix domain="mail.pl" /> - <suffix domain="miasta.pl" /> - <suffix domain="media.pl" /> - <suffix domain="mil.pl" /> - <suffix domain="net.pl" /> - <suffix domain="nieruchomosci.pl" /> - <suffix domain="nom.pl" /> - <suffix domain="org.pl" /> - <suffix domain="pc.pl" /> - <suffix domain="powiat.pl" /> - <suffix domain="priv.pl" /> - <suffix domain="realestate.pl" /> - <suffix domain="rel.pl" /> - <suffix domain="sex.pl" /> - <suffix domain="shop.pl" /> - <suffix domain="sklep.pl" /> - <suffix domain="sos.pl" /> - <suffix domain="szkola.pl" /> - <suffix domain="targi.pl" /> - <suffix domain="tm.pl" /> - <suffix domain="tourism.pl" /> - <suffix domain="travel.pl" /> - <suffix domain="turystyka.pl" /> - <!-- ICM functional domains (icm.edu.pl)--> - <suffix domain="6bone.pl" /> - <suffix domain="art.pl" /> - <suffix domain="mbone.pl" /> - <!-- Government domains (administred by ippt.gov.pl)--> - <suffix domain="gov.pl" /> - <suffix domain="uw.gov.pl" /> - <suffix domain="um.gov.pl" /> - <suffix domain="ug.gov.pl" /> - <suffix domain="upow.gov.pl" /> - <suffix domain="starostwo.gov.pl" /> - <suffix domain="so.gov.pl" /> - <suffix domain="sr.gov.pl" /> - <suffix domain="po.gov.pl" /> - <suffix domain="pa.gov.pl" /> - <!-- other functional domains--> - <suffix domain="med.pl" /> - <suffix domain="ngo.pl" /> - <suffix domain="irc.pl" /> - <suffix domain="usenet.pl" /> - <!-- NASK geographical domains : http://www.dns.pl/english/dns-regiony.html--> - <suffix domain="augustow.pl" /> - <suffix domain="babia-gora.pl" /> - <suffix domain="bedzin.pl" /> - <suffix domain="beskidy.pl" /> - <suffix domain="bialowieza.pl" /> - <suffix domain="bialystok.pl" /> - <suffix domain="bielawa.pl" /> - <suffix domain="bieszczady.pl" /> - <suffix domain="boleslawiec.pl" /> - <suffix domain="bydgoszcz.pl" /> - <suffix domain="bytom.pl" /> - <suffix domain="cieszyn.pl" /> - <suffix domain="czeladz.pl" /> - <suffix domain="czest.pl" /> - <suffix domain="dlugoleka.pl" /> - <suffix domain="elblag.pl" /> - <suffix domain="elk.pl" /> - <suffix domain="glogow.pl" /> - <suffix domain="gniezno.pl" /> - <suffix domain="gorlice.pl" /> - <suffix domain="grajewo.pl" /> - <suffix domain="ilawa.pl" /> - <suffix domain="jaworzno.pl" /> - <suffix domain="jelenia-gora.pl" /> - <suffix domain="jgora.pl" /> - <suffix domain="kalisz.pl" /> - <suffix domain="kazimierz-dolny.pl" /> - <suffix domain="karpacz.pl" /> - <suffix domain="kartuzy.pl" /> - <suffix domain="kaszuby.pl" /> - <suffix domain="katowice.pl" /> - <suffix domain="kepno.pl" /> - <suffix domain="ketrzyn.pl" /> - <suffix domain="klodzko.pl" /> - <suffix domain="kobierzyce.pl" /> - <suffix domain="kolobrzeg.pl" /> - <suffix domain="konin.pl" /> - <suffix domain="konskowola.pl" /> - <suffix domain="kutno.pl" /> - <suffix domain="lapy.pl" /> - <suffix domain="lebork.pl" /> - <suffix domain="legnica.pl" /> - <suffix domain="lezajsk.pl" /> - <suffix domain="limanowa.pl" /> - <suffix domain="lomza.pl" /> - <suffix domain="lowicz.pl" /> - <suffix domain="lubin.pl" /> - <suffix domain="lukow.pl" /> - <suffix domain="malbork.pl" /> - <suffix domain="malopolska.pl" /> - <suffix domain="mazowsze.pl" /> - <suffix domain="mazury.pl" /> - <suffix domain="mielec.pl" /> - <suffix domain="mielno.pl" /> - <suffix domain="mragowo.pl" /> - <suffix domain="naklo.pl" /> - <suffix domain="nowaruda.pl" /> - <suffix domain="nysa.pl" /> - <suffix domain="olawa.pl" /> - <suffix domain="olecko.pl" /> - <suffix domain="olkusz.pl" /> - <suffix domain="olsztyn.pl" /> - <suffix domain="opoczno.pl" /> - <suffix domain="opole.pl" /> - <suffix domain="ostroda.pl" /> - <suffix domain="ostroleka.pl" /> - <suffix domain="ostrowiec.pl" /> - <suffix domain="ostrowwlkp.pl" /> - <suffix domain="pila.pl" /> - <suffix domain="pisz.pl" /> - <suffix domain="podhale.pl" /> - <suffix domain="podlasie.pl" /> - <suffix domain="polkowice.pl" /> - <suffix domain="pomorze.pl" /> - <suffix domain="pomorskie.pl" /> - <suffix domain="prochowice.pl" /> - <suffix domain="pruszkow.pl" /> - <suffix domain="przeworsk.pl" /> - <suffix domain="pulawy.pl" /> - <suffix domain="radom.pl" /> - <suffix domain="rawa-maz.pl" /> - <suffix domain="rybnik.pl" /> - <suffix domain="rzeszow.pl" /> - <suffix domain="sanok.pl" /> - <suffix domain="sejny.pl" /> - <suffix domain="slask.pl" /> - <suffix domain="slupsk.pl" /> - <suffix domain="sosnowiec.pl" /> - <suffix domain="stalowa-wola.pl" /> - <suffix domain="skoczow.pl" /> - <suffix domain="starachowice.pl" /> - <suffix domain="stargard.pl" /> - <suffix domain="suwalki.pl" /> - <suffix domain="swidnica.pl" /> - <suffix domain="swiebodzin.pl" /> - <suffix domain="swinoujscie.pl" /> - <suffix domain="szczecin.pl" /> - <suffix domain="szczytno.pl" /> - <suffix domain="tarnobrzeg.pl" /> - <suffix domain="tgory.pl" /> - <suffix domain="turek.pl" /> - <suffix domain="tychy.pl" /> - <suffix domain="ustka.pl" /> - <suffix domain="walbrzych.pl" /> - <suffix domain="warmia.pl" /> - <suffix domain="warszawa.pl" /> - <suffix domain="waw.pl" /> - <suffix domain="wegrow.pl" /> - <suffix domain="wielun.pl" /> - <suffix domain="wlocl.pl" /> - <suffix domain="wloclawek.pl" /> - <suffix domain="wodzislaw.pl" /> - <suffix domain="wolomin.pl" /> - <suffix domain="wroclaw.pl" /> - <suffix domain="zachpomor.pl" /> - <suffix domain="zagan.pl" /> - <suffix domain="zarow.pl" /> - <suffix domain="zgora.pl" /> - <suffix domain="zgorzelec.pl" /> - <!-- TASK geographical domains (www.task.gda.pl/uslugi/dns)--> - <suffix domain="gda.pl" /> - <suffix domain="gdansk.pl" /> - <suffix domain="gdynia.pl" /> - <suffix domain="sopot.pl" /> - <!-- other geographical domains--> - <suffix domain="gliwice.pl" /> - <suffix domain="krakow.pl" /> - <suffix domain="poznan.pl" /> - <suffix domain="wroc.pl" /> - <suffix domain="zakopane.pl" /> - - <!-- pn : http://www.government.pn/PnRegistry/policies.htm--> - <suffix domain="gov.pn" /> - <suffix domain="co.pn" /> - <suffix domain="org.pn" /> - <suffix domain="edu.pn" /> - <suffix domain="net.pn" /> - - <!-- pr : http://www.nic.pr/index.asp?f=1--> - <suffix domain="com.pr" /> - <suffix domain="net.pr" /> - <suffix domain="org.pr" /> - <suffix domain="gov.pr" /> - <suffix domain="edu.pr" /> - <suffix domain="isla.pr" /> - <suffix domain="pro.pr" /> - <suffix domain="biz.pr" /> - <suffix domain="info.pr" /> - <suffix domain="name.pr" /> - <!-- these aren't mentioned on nic.pr, but on http://en.wikipedia.org/wiki/.pr--> - <suffix domain="est.pr" /> - <suffix domain="prof.pr" /> - <suffix domain="ac.pr" /> - - <!-- pro : http://www.nic.pro/support_faq.htm--> - <suffix domain="aca.pro" /> - <suffix domain="bar.pro" /> - <suffix domain="cpa.pro" /> - <suffix domain="jur.pro" /> - <suffix domain="law.pro" /> - <suffix domain="med.pro" /> - <suffix domain="eng.pro" /> - - <!-- ps : http://en.wikipedia.org/wiki/.ps--> - <!-- list of 2nd level tlds ?--> - <suffix domain="edu.ps" /> - <suffix domain="gov.ps" /> - <suffix domain="sec.ps" /> - <suffix domain="plo.ps" /> - <suffix domain="com.ps" /> - <suffix domain="org.ps" /> - <suffix domain="net.ps" /> - - <!-- pt : http://online.dns.pt/dns/start_dns--> - <suffix domain="net.pt" /> - <suffix domain="gov.pt" /> - <suffix domain="org.pt" /> - <suffix domain="edu.pt" /> - <suffix domain="int.pt" /> - <suffix domain="publ.pt" /> - <suffix domain="com.pt" /> - <suffix domain="nome.pt" /> - - <!-- pw : http://en.wikipedia.org/wiki/.pw--> - - <!-- py : http://www.nic.py/faq_a.html#faq_b--> - - <!-- qa : http://www.qatar.net.qa/services/virtual.htm--> - - <!-- re : http://www.afnic.re/obtenir/chartes/nommage-re/annexe-descriptifs--> - <suffix domain="com.re" /> - <suffix domain="asso.re" /> - <suffix domain="nom.re" /> - - <!-- ro : http://www.rotld.ro/--> - <suffix domain="com.ro" /> - <suffix domain="org.ro" /> - <suffix domain="tm.ro" /> - <suffix domain="nt.ro" /> - <suffix domain="nom.ro" /> - <suffix domain="info.ro" /> - <suffix domain="rec.ro" /> - <suffix domain="arts.ro" /> - <suffix domain="firm.ro" /> - <suffix domain="store.ro" /> - <suffix domain="www.ro" /> - - <!-- ru : http://en.wikipedia.org/wiki/.ru--> - <suffix domain="com.ru" /> - <suffix domain="net.ru" /> - <suffix domain="org.ru" /> - <suffix domain="pp.ru" /> - <suffix domain="int.ru" /> - <!-- there should be geo-names like msk.ru, but I didn't find a list--> - - <!-- rw : http://www.nic.rw/cgi-bin/policy.pl--> - <suffix domain="gov.rw" /> - <suffix domain="net.rw" /> - <suffix domain="edu.rw" /> - <suffix domain="ac.rw" /> - <suffix domain="com.rw" /> - <suffix domain="co.rw" /> - <suffix domain="int.rw" /> - <suffix domain="mil.rw" /> - <suffix domain="gouv.rw" /> - - <!-- sa : http://www.saudinic.net.sa/page.php?page=1&lang=1--> - - <!-- sb : http://www.sbnic.net.sb/--> - - <!-- sc : http://www.nic.sc/--> - <suffix domain="com.sc" /> - <suffix domain="gov.sc" /> - <suffix domain="net.sc" /> - <suffix domain="org.sc" /> - <suffix domain="edu.sc" /> - - <!-- sd : http://www.isoc.sd/sudanic.isoc.sd/billing_pricing.htm--> - <suffix domain="com.sd" /> - <suffix domain="net.sd" /> - <suffix domain="org.sd" /> - <suffix domain="edu.sd" /> - <suffix domain="med.sd" /> - <suffix domain="tv.sd" /> - <suffix domain="gov.sd" /> - <suffix domain="info.sd" /> - - <!-- se : http://en.wikipedia.org/wiki/.se--> - <suffix domain="org.se" /> - <suffix domain="pp.se" /> - <suffix domain="tm.se" /> - <suffix domain="parti.se" /> - <suffix domain="press.se" /> - <suffix domain="mil.se" /> - <!-- se geographical names--> - <suffix domain="ab.se" /> - <suffix domain="c.se" /> - <suffix domain="d.se" /> - <suffix domain="e.se" /> - <suffix domain="f.se" /> - <suffix domain="g.se" /> - <suffix domain="h.se" /> - <suffix domain="i.se" /> - <suffix domain="k.se" /> - <suffix domain="m.se" /> - <suffix domain="n.se" /> - <suffix domain="o.se" /> - <suffix domain="s.se" /> - <suffix domain="t.se" /> - <suffix domain="u.se" /> - <suffix domain="w.se" /> - <suffix domain="x.se" /> - <suffix domain="y.se" /> - <suffix domain="z.se" /> - <suffix domain="ac.se" /> - <suffix domain="bd.se" /> - - <!-- sg : http://www.nic.net.sg/sub_policies_agreement/2ld.html--> - <suffix domain="com.sg" /> - <suffix domain="net.sg" /> - <suffix domain="org.sg" /> - <suffix domain="gov.sg" /> - <suffix domain="edu.sg" /> - <suffix domain="per.sg" /> - - <!-- sh : http://www.nic.sh/rules.html--> - <!-- list of 2nd level domains ?--> - - <!-- si : http://en.wikipedia.org/wiki/.si--> - - <!-- sk : http://en.wikipedia.org/wiki/.sk--> - - <!-- sl : http://en.wikipedia.org/wiki/.sl--> - <!-- list of 2nd level domains ?--> - - <!-- sm : http://en.wikipedia.org/wiki/.sm--> - - <!-- sn : http://en.wikipedia.org/wiki/.sn--> - <!-- list of 2nd level domains ?--> - - <!-- sr : http://en.wikipedia.org/wiki/.sr--> - - <!-- st : http://www.nic.st/html/policyrules/--> - - <!-- su : http://en.wikipedia.org/wiki/.su--> - - <!-- sv : http://www.svnet.org.sv/svpolicy.html--> - - <!-- sy : http://www.gobin.info/domainname/sy.doc--> - - <!-- sz : http://en.wikipedia.org/wiki/.sz--> - <!-- list of 2nd level domains ?--> - - <!-- tc : http://en.wikipedia.org/wiki/.tc--> - - <!-- td : http://en.wikipedia.org/wiki/.td--> - - <!-- tf : http://en.wikipedia.org/wiki/.tf--> - - <!-- tg : http://en.wikipedia.org/wiki/.tg--> - <!-- list of 2nd level domains ?--> - - <!-- th : http://en.wikipedia.org/wiki/.th--> - - <!-- tj : http://www.nic.tj/policy.htm--> - <suffix domain="ac.tj" /> - <suffix domain="biz.tj" /> - <suffix domain="com.tj" /> - <suffix domain="co.tj" /> - <suffix domain="edu.tj" /> - <suffix domain="int.tj" /> - <suffix domain="name.tj" /> - <suffix domain="net.tj" /> - <suffix domain="org.tj" /> - <suffix domain="web.tj" /> - <suffix domain="gov.tj" /> - <suffix domain="go.tj" /> - <suffix domain="mil.tj" /> - - <!-- tk : http://en.wikipedia.org/wiki/.tk--> - - <!-- tl : http://en.wikipedia.org/wiki/.tl--> - <!-- list of 2nd level tlds ?--> - - <!-- tm : http://www.nic.tm/rules.html--> - <!-- list of 2nd level tlds ?--> - - <!-- tn : http://en.wikipedia.org/wiki/.tn--> - <!-- list of 2nd level tlds ?--> - - <!-- to : http://en.wikipedia.org/wiki/.to--> - <!-- list of 2nd level tlds ?--> - - <!-- tr : http://en.wikipedia.org/wiki/.tr--> - - <!-- travel : http://en.wikipedia.org/wiki/.travel--> - - <!-- tt : http://www.nic.tt/--> - <suffix domain="co.tt" /> - <suffix domain="com.tt" /> - <suffix domain="org.tt" /> - <suffix domain="net.tt" /> - <suffix domain="biz.tt" /> - <suffix domain="info.tt" /> - <suffix domain="pro.tt" /> - <suffix domain="int.tt" /> - <suffix domain="coop.tt" /> - <suffix domain="jobs.tt" /> - <suffix domain="mobi.tt" /> - <suffix domain="travel.tt" /> - <suffix domain="museum.tt" /> - <suffix domain="aero.tt" /> - <suffix domain="name.tt" /> - <suffix domain="gov.tt" /> - <suffix domain="edu.tt" /> - - <!-- tv : http://en.wikipedia.org/wiki/.tv--> - <!-- list of 2nd level tlds ?--> - - <!-- tw : http://en.wikipedia.org/wiki/.tw--> - <suffix domain="edu.tw" /> - <suffix domain="gov.tw" /> - <suffix domain="mil.tw" /> - <suffix domain="com.tw" /> - <suffix domain="net.tw" /> - <suffix domain="org.tw" /> - <suffix domain="idv.tw" /> - <suffix domain="game.tw" /> - <suffix domain="ebiz.tw" /> - <suffix domain="club.tw" /> - <suffix domain="網路.tw" /> - <suffix domain="組織.tw" /> - <suffix domain="商æ¥Â.tw" /> - - <!-- tz : http://en.wikipedia.org/wiki/.tz--> - - <!-- ua : http://www.nic.net.ua/--> - <suffix domain="com.ua" /> - <suffix domain="edu.ua" /> - <suffix domain="gov.ua" /> - <suffix domain="net.ua" /> - <suffix domain="org.ua" /> - <!-- ua geo-names--> - <suffix domain="cherkassy.ua" /> - <suffix domain="chernigov.ua" /> - <suffix domain="chernovtsy.ua" /> - <suffix domain="ck.ua" /> - <suffix domain="cn.ua" /> - <suffix domain="crimea.ua" /> - <suffix domain="cv.ua" /> - <suffix domain="dn.ua" /> - <suffix domain="dnepropetrovsk.ua" /> - <suffix domain="donetsk.ua" /> - <suffix domain="dp.ua" /> - <suffix domain="if.ua" /> - <suffix domain="ivano-frankivsk.ua" /> - <suffix domain="kh.ua" /> - <suffix domain="kharkov.ua" /> - <suffix domain="kherson.ua" /> - <suffix domain="kiev.ua" /> - <suffix domain="kirovograd.ua" /> - <suffix domain="km.ua" /> - <suffix domain="kr.ua" /> - <suffix domain="ks.ua" /> - <suffix domain="lg.ua" /> - <suffix domain="lugansk.ua" /> - <suffix domain="lutsk.ua" /> - <suffix domain="lviv.ua" /> - <suffix domain="mk.ua" /> - <suffix domain="nikolaev.ua" /> - <suffix domain="od.ua" /> - <suffix domain="odessa.ua" /> - <suffix domain="pl.ua" /> - <suffix domain="poltava.ua" /> - <suffix domain="rovno.ua" /> - <suffix domain="rv.ua" /> - <suffix domain="sebastopol.ua" /> - <suffix domain="sumy.ua" /> - <suffix domain="te.ua" /> - <suffix domain="ternopil.ua" /> - <suffix domain="vinnica.ua" /> - <suffix domain="vn.ua" /> - <suffix domain="zaporizhzhe.ua" /> - <suffix domain="zp.ua" /> - <suffix domain="uz.ua" /> - <suffix domain="uzhgorod.ua" /> - <suffix domain="zhitomir.ua" /> - <suffix domain="zt.ua" /> - - <!-- ug : http://www.registry.co.ug/--> - <suffix domain="co.ug" /> - <suffix domain="ac.ug" /> - <suffix domain="sc.ug" /> - <suffix domain="go.ug" /> - <suffix domain="ne.ug" /> - <suffix domain="or.ug" /> - - <!-- uk : http://en.wikipedia.org/wiki/.uk--> - <suffix domain="sch.uk" /> - <suffix domain="bl.uk" /> - <suffix domain="british-library.uk" /> - <suffix domain="icnet.uk" /> - <suffix domain="jet.uk" /> - <suffix domain="nel.uk" /> - <suffix domain="nls.uk" /> - <suffix domain="national-library-scotland.uk" /> - <suffix domain="parliament.uk" /> - - <!-- us : http://en.wikipedia.org/wiki/.us--> - <suffix domain="dni.us" /> - <suffix domain="fed.us" /> - <suffix domain="isa.us" /> - <suffix domain="kids.us" /> - <suffix domain="nsn.us" /> - <!-- us geographic names--> - <suffix domain="ak.us" /> - <suffix domain="al.us" /> - <suffix domain="ar.us" /> - <suffix domain="az.us" /> - <suffix domain="ca.us" /> - <suffix domain="co.us" /> - <suffix domain="ct.us" /> - <suffix domain="dc.us" /> - <suffix domain="de.us" /> - <suffix domain="fl.us" /> - <suffix domain="ga.us" /> - <suffix domain="hi.us" /> - <suffix domain="ia.us" /> - <suffix domain="id.us" /> - <suffix domain="il.us" /> - <suffix domain="in.us" /> - <suffix domain="ks.us" /> - <suffix domain="ky.us" /> - <suffix domain="la.us" /> - <suffix domain="ma.us" /> - <suffix domain="md.us" /> - <suffix domain="me.us" /> - <suffix domain="mi.us" /> - <suffix domain="mn.us" /> - <suffix domain="mo.us" /> - <suffix domain="ms.us" /> - <suffix domain="mt.us" /> - <suffix domain="nc.us" /> - <suffix domain="nd.us" /> - <suffix domain="ne.us" /> - <suffix domain="nh.us" /> - <suffix domain="nj.us" /> - <suffix domain="nm.us" /> - <suffix domain="nv.us" /> - <suffix domain="ny.us" /> - <suffix domain="oh.us" /> - <suffix domain="ok.us" /> - <suffix domain="or.us" /> - <suffix domain="pa.us" /> - <suffix domain="ri.us" /> - <suffix domain="sc.us" /> - <suffix domain="sd.us" /> - <suffix domain="tn.us" /> - <suffix domain="tx.us" /> - <suffix domain="ut.us" /> - <suffix domain="vt.us" /> - <suffix domain="va.us" /> - <suffix domain="wa.us" /> - <suffix domain="wi.us" /> - <suffix domain="wv.us" /> - <suffix domain="wy.us" /> - <!-- the following rules would be only valid under the geo-name, but we can't express that--> - <!-- *.*.us cities, counties, parishes, and townships (locality.state.us)--> - <!-- !ci.*.*.us city government agencies (subdomain under locality)--> - <!-- !town.*.*.us town government agencies (subdomain under locality)--> - <!-- !co.*.*.us county government agencies (subdomain under locality)--> - <!-- k12.*.us public school districts--> - <!-- pvt.k12.*.us private schools--> - <!-- cc.*.us community colleges--> - <!-- tec.*.us technical and vocational schools--> - <!-- lib.*.us state, regional, city, and county libraries--> - <!-- state.*.us state government agencies--> - <!-- gen.*.us general independent entities (groups not fitting into the above categories)--> - - <!-- uy : http://www.antel.com.uy/--> - - <!-- uz : http://www.reg.uz/registerr.html--> - <!-- are there other 2nd level tlds ?--> - <suffix domain="com.uz" /> - <suffix domain="co.uz" /> - - <!-- va : http://en.wikipedia.org/wiki/.va--> - - <!-- vc : http://en.wikipedia.org/wiki/.vc--> - <!-- list of 2nd level tlds ?--> - - <!-- ve : http://registro.nic.ve/nicve/registro/index.html--> - - <!-- vg : http://en.wikipedia.org/wiki/.vg--> - - <!-- vi : http://www.nic.vi/Domain_Rules/body_domain_rules.html--> - <suffix domain="com.vi" /> - <suffix domain="org.vi" /> - <suffix domain="edu.vi" /> - <suffix domain="gov.vi" /> - - <!-- vn : https://www.dot.vn/vnnic/vnnic/domainregistration.jsp--> - <suffix domain="com.vn" /> - <suffix domain="net.vn" /> - <suffix domain="org.vn" /> - <suffix domain="edu.vn" /> - <suffix domain="gov.vn" /> - <suffix domain="int.vn" /> - <suffix domain="ac.vn" /> - <suffix domain="biz.vn" /> - <suffix domain="info.vn" /> - <suffix domain="name.vn" /> - <suffix domain="pro.vn" /> - <suffix domain="health.vn" /> - - <!-- vu : http://en.wikipedia.org/wiki/.vu--> - <!-- list of 2nd level tlds ?--> - - <!-- ws : http://en.wikipedia.org/wiki/.ws--> - - <!-- ye : http://www.y.net.ye/services/domain_name.htm--> - - <!-- yu : http://www.nic.yu/pravilnik-e.html--> - - <!-- za : http://www.zadna.org.za/slds.html--> - - <!-- zm : http://en.wikipedia.org/wiki/.zm--> - - <!-- zw : http://en.wikipedia.org/wiki/.zw--> - - </suffixes> -</domains> diff --git a/search-server/nutch/crawl_jsp/conf/domain-suffixes.xsd b/search-server/nutch/crawl_jsp/conf/domain-suffixes.xsd deleted file mode 100644 index ba68d62..0000000 --- a/search-server/nutch/crawl_jsp/conf/domain-suffixes.xsd +++ /dev/null @@ -1,130 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- - Document : domain-suffixes.xsd - Author : Enis Soztutar - enis.soz.nutch@gmail.com - Description: This document is the schema for valid domain-suffixes - definitions. For successful parsing of domain-suffixes xml files, - the xml file should be validated with this xsd. - See : org.apache.nutch.util.domain.DomainSuffixesReader.java ---> - -<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" - targetNamespace="http://lucene.apache.org/nutch" - xmlns="http://lucene.apache.org/nutch" - elementFormDefault="qualified"> - - <xs:element name="domains"> - xs:complexType - xs:sequence - <xs:element name="tlds"> - xs:complexType - xs:sequence - <xs:element name="itlds"> - xs:complexType - xs:sequence - <xs:element name="tld" maxOccurs="unbounded" - type="gtld" /> - </xs:sequence> - </xs:complexType> - </xs:element> - - <xs:element name="gtlds"> - xs:complexType - xs:sequence - <xs:element name="tld" maxOccurs="unbounded" - type="gtld" /> - </xs:sequence> - </xs:complexType> - </xs:element> - - <xs:element name="cctlds"> - xs:complexType - xs:sequence - <xs:element name="tld" maxOccurs="unbounded" - type="cctld" /> - </xs:sequence> - </xs:complexType> - </xs:element> - - </xs:sequence> - </xs:complexType> - </xs:element> - - <xs:element name="suffixes"> - xs:complexType - xs:sequence - <xs:element name="suffix" maxOccurs="unbounded" - type="sldType" /> - </xs:sequence> - </xs:complexType> - </xs:element> - </xs:sequence> - </xs:complexType> - </xs:element> - - <xs:complexType name="gtld"> - xs:sequence - <xs:element name="status" minOccurs="0"> - xs:simpleType - <xs:restriction base="xs:string"> - <xs:enumeration value="INFRASTRUCTURE" /> - <xs:enumeration value="SPONSORED" /> - <xs:enumeration value="UNSPONSORED" /> - <xs:enumeration value="STARTUP" /> - <xs:enumeration value="PROPOSED" /> - <xs:enumeration value="DELETED" /> - <xs:enumeration value="PSEUDO_DOMAIN" /> - </xs:restriction> - </xs:simpleType> - </xs:element> - <xs:element name="boost" type="xs:float" minOccurs="0" /> - <xs:element name="description" type="xs:string" minOccurs="0" /> - </xs:sequence> - <xs:attribute name="domain" type="xs:string" /> - </xs:complexType> - - <xs:complexType name="cctld"> - xs:sequence - <xs:element name="country" type="xs:string" /> - <xs:element name="status" type="statusType" minOccurs="0" /> - <xs:element name="boost" type="xs:float" minOccurs="0" /> - <xs:element name="description" type="xs:string" minOccurs="0" /> - </xs:sequence> - <xs:attribute name="domain" type="xs:string" /> - </xs:complexType> - - <xs:complexType name="sldType"> - xs:sequence - <xs:element name="status" type="statusType" minOccurs="0" /> - <xs:element name="boost" type="xs:float" minOccurs="0" /> - <xs:element name="description" type="xs:string" minOccurs="0" /> - </xs:sequence> - <xs:attribute name="domain" type="xs:string" /> - </xs:complexType> - - <xs:simpleType name="statusType"> - <xs:restriction base="xs:string"> - <xs:enumeration value="IN_USE" /> - <xs:enumeration value="NOT_IN_USE" /> - <xs:enumeration value="DELETED" /> - </xs:restriction> - </xs:simpleType> - -</xs:schema> diff --git a/search-server/nutch/crawl_jsp/conf/hadoop-env.sh b/search-server/nutch/crawl_jsp/conf/hadoop-env.sh deleted file mode 100644 index 9050a6e..0000000 --- a/search-server/nutch/crawl_jsp/conf/hadoop-env.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Set Hadoop-specific environment variables here. - -# The only required environment variable is JAVA_HOME. All others are -# optional. When running a distributed configuration it is best to -# set JAVA_HOME in this file, so that it is correctly defined on -# remote nodes. - -# The java implementation to use. Required. -# export JAVA_HOME=/usr/bin/java - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HADOOP_HEAPSIZE=2000 - -# Extra Java runtime options. Empty by default. -# export HADOOP_OPTS=-server - -# Extra ssh options. Default: '-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR'. -# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR" - -# Where log files are stored. $HADOOP_HOME/logs by default. -# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs - -# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. -# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves - -# host:path where hadoop code should be rsync'd from. Unset by default. -# export HADOOP_MASTER=master:/home/$USER/src/hadoop - -# The directory where pid files are stored. /tmp by default. -# export HADOOP_PID_DIR=/var/hadoop/pids - -# A string representing this instance of hadoop. $USER by default. -# export HADOOP_IDENT_STRING=$USER diff --git a/search-server/nutch/crawl_jsp/conf/hadoop-site.xml b/search-server/nutch/crawl_jsp/conf/hadoop-site.xml deleted file mode 100644 index 970c8fe..0000000 --- a/search-server/nutch/crawl_jsp/conf/hadoop-site.xml +++ /dev/null @@ -1,8 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<!-- Put site-specific property overrides in this file. --> - -<configuration> - -</configuration> diff --git a/search-server/nutch/crawl_jsp/conf/httpclient-auth.xml b/search-server/nutch/crawl_jsp/conf/httpclient-auth.xml deleted file mode 100644 index 60d45b9..0000000 --- a/search-server/nutch/crawl_jsp/conf/httpclient-auth.xml +++ /dev/null @@ -1,61 +0,0 @@ -<?xml version="1.0"?> -<!-- - This is the authentication configuration file for protocol-httpclient. - Different credentials for different authentication scopes can be - configured in this file. If a set of credentials is configured for a - particular authentication scope (i.e. particular host, port number, - scheme and realm), then that set of credentials would be sent only to - servers falling under the specified authentication scope. Apart from - this at most one set of credentials can be configured as 'default'. - - When authentication is required to fetch a resource from a web-server, - the authentication-scope is determined from the host, port, scheme and - realm (if present) obtained from the URL of the page and the - authentication headers in the HTTP response. If it matches any - 'authscope' in this configuration file, then the 'credentials' for - that 'authscope' is used for authentication. Otherwise, it would use - the 'default' set of credentials (with an exception which is described - in the next paragraph), if present. If any attribute is missing, it - would match all values for that attribute. - - If there are several pages having different authentication realms and - schemes on the same web-server (same host and port, but different - realms and schemes), and credentials for one or more of the realms and - schemes for that web-server is specified, then the 'default' - credentials would be ignored completely for that web-server (for that - host and port). So, credentials to handle all realms and schemes for - that server may be specified explicitly by adding an extra 'authscope' - tag with the 'realm' and 'scheme' attributes missing for that server. - This is demonstrated by the last 'authscope' tag for 'example:8080' in - the following example. - - Example:- - <credentials username="susam" password="masus"> - <default realm="sso"/> - <authscope host="192.168.101.33" port="80" realm="login"/> - <authscope host="example" port="8080" realm="blogs"/> - <authscope host="example" port="8080" realm="wiki"/> - <authscope host="example" port="80" realm="quiz" scheme="NTLM"/> - </credentials> - <credentials username="admin" password="nimda"> - <authscope host="example" port="8080"/> - </credentials> - - In the above example, 'example:8080' server has pages with multiple - authentication realms. The first set of credentials would be used for - 'blogs' and 'wiki' authentication realms. The second set of - credentials would be used for all other realms. For 'login' realm of - '192.168.101.33', the first set of credentials would be used. For any - other realm of '192.168.101.33' authentication would not be done. For - the NTLM authentication required by 'example:80', the first set of - credentials would be used. For 'sso' realms of all other servers, the - first set of credentials would be used, since it is configured as - 'default'. - - NTLM does not use the notion of realms. The domain name may be - specified as the value for 'realm' attribute in case of NTLM. ---> - -<auth-configuration> - -</auth-configuration> diff --git a/search-server/nutch/crawl_jsp/conf/log4j.properties b/search-server/nutch/crawl_jsp/conf/log4j.properties deleted file mode 100644 index 5a985ab..0000000 --- a/search-server/nutch/crawl_jsp/conf/log4j.properties +++ /dev/null @@ -1,82 +0,0 @@ -# Define some default values that can be overridden by system properties -hadoop.log.dir=. -hadoop.log.file=hadoop.log - -# RootLogger - DailyRollingFileAppender -log4j.rootLogger=ALL,DRFA - -# Logging Threshold -log4j.threshhold=ALL - -#special logging requirements for some commandline tools -log4j.logger.org.apache.nutch.crawl.Crawl=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.Injector=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.Generator=INFO,cmdstdout -log4j.logger.org.apache.nutch.fetcher.Fetcher=INFO,cmdstdout -log4j.logger.org.apache.nutch.parse.ParseSegment=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.CrawlDbReader=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.CrawlDbMerger=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.LinkDbReader=INFO,cmdstdout -log4j.logger.org.apache.nutch.segment.SegmentReader=INFO,cmdstdout -log4j.logger.org.apache.nutch.segment.SegmentMerger=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.CrawlDb=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.LinkDb=INFO,cmdstdout -log4j.logger.org.apache.nutch.crawl.LinkDbMerger=INFO,cmdstdout -log4j.logger.org.apache.nutch.indexer.Indexer=INFO,cmdstdout -log4j.logger.org.apache.nutch.indexer.DeleteDuplicates=INFO,cmdstdout -log4j.logger.org.apache.nutch.indexer.IndexMerger=INFO,cmdstdout - -log4j.logger.org.apache.nutch=INFO -log4j.logger.org.apache.hadoop=WARN - -# -# Daily Rolling File Appender -# - -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n -# Debugging Pattern format: Date LogLevel LoggerName (FileName:MethodName:LineNo) LogMessage -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# -# stdout -# Add *stdout* to rootlogger above if you want to use this -# - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - -# -# plain layout used for commandline tools to output to console -# -log4j.appender.cmdstdout=org.apache.log4j.ConsoleAppender -log4j.appender.cmdstdout.layout=org.apache.log4j.PatternLayout -log4j.appender.cmdstdout.layout.ConversionPattern=%m%n - -# -# Rolling File Appender -# - -#log4j.appender.RFA=org.apache.log4j.RollingFileAppender -#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Logfile size and and 30-day backups -#log4j.appender.RFA.MaxFileSize=1MB -#log4j.appender.RFA.MaxBackupIndex=30 - -#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout -#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n -#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - diff --git a/search-server/nutch/crawl_jsp/conf/logging.properties b/search-server/nutch/crawl_jsp/conf/logging.properties deleted file mode 100644 index 67b4a73..0000000 --- a/search-server/nutch/crawl_jsp/conf/logging.properties +++ /dev/null @@ -1,4 +0,0 @@ -handlers= java.util.logging.ConsoleHandler -.level= FINE - - diff --git a/search-server/nutch/crawl_jsp/conf/masters b/search-server/nutch/crawl_jsp/conf/masters deleted file mode 100644 index 2fbb50c..0000000 --- a/search-server/nutch/crawl_jsp/conf/masters +++ /dev/null @@ -1 +0,0 @@ -localhost diff --git a/search-server/nutch/crawl_jsp/conf/nutch-conf.xsl b/search-server/nutch/crawl_jsp/conf/nutch-conf.xsl deleted file mode 100644 index 36a2275..0000000 --- a/search-server/nutch/crawl_jsp/conf/nutch-conf.xsl +++ /dev/null @@ -1,24 +0,0 @@ -<?xml version="1.0"?> -<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> -<xsl:output method="html"/> -<xsl:template match="nutch-conf"> -<html> -<body> -<table border="1"> -<tr> - <td>name</td> - <td>value</td> - <td>description</td> -</tr> -<xsl:for-each select="property"> -<tr> - <td><xsl:value-of select="name"/></td> - <td><xsl:value-of select="value"/></td> - <td><xsl:value-of select="description"/></td> -</tr> -</xsl:for-each> -</table> -</body> -</html> -</xsl:template> -</xsl:stylesheet> diff --git a/search-server/nutch/crawl_jsp/conf/nutch-default.xml b/search-server/nutch/crawl_jsp/conf/nutch-default.xml deleted file mode 100644 index 2bf0e1e..0000000 --- a/search-server/nutch/crawl_jsp/conf/nutch-default.xml +++ /dev/null @@ -1,1205 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<!-- Do not modify this file directly. Instead, copy entries that you --> -<!-- wish to modify from this file into nutch-site.xml and change them --> -<!-- there. If nutch-site.xml does not already exist, create it. --> - -<configuration> - -<!-- file properties --> - -<property> - <name>file.content.limit</name> - <value>65536</value> - <description>The length limit for downloaded content, in bytes. - If this value is nonnegative (>=0), content longer than it will be truncated; - otherwise, no truncation at all. - </description> -</property> - -<property> - <name>file.content.ignored</name> - <value>true</value> - <description>If true, no file content will be saved during fetch. - And it is probably what we want to set most of time, since file:// URLs - are meant to be local and we can always use them directly at parsing - and indexing stages. Otherwise file contents will be saved. - !! NO IMPLEMENTED YET !! - </description> -</property> - -<!-- HTTP properties --> - -<property> - <name>http.agent.name</name> - <value></value> - <description>HTTP 'User-Agent' request header. MUST NOT be empty - - please set this to a single word uniquely related to your organization. - - NOTE: You should also check other related properties: - - http.robots.agents - http.agent.description - http.agent.url - http.agent.email - http.agent.version - - and set their values appropriately. - - </description> -</property> - -<property> - <name>http.robots.agents</name> - <value>*</value> - <description>The agent strings we'll look for in robots.txt files, - comma-separated, in decreasing order of precedence. You should - put the value of http.agent.name as the first agent name, and keep the - default * at the end of the list. E.g.: BlurflDev,Blurfl,* - </description> -</property> - -<property> - <name>http.robots.403.allow</name> - <value>true</value> - <description>Some servers return HTTP status 403 (Forbidden) if - /robots.txt doesn't exist. This should probably mean that we are - allowed to crawl the site nonetheless. If this is set to false, - then such sites will be treated as forbidden.</description> -</property> - -<property> - <name>http.agent.description</name> - <value></value> - <description>Further description of our bot- this text is used in - the User-Agent header. It appears in parenthesis after the agent name. - </description> -</property> - -<property> - <name>http.agent.url</name> - <value></value> - <description>A URL to advertise in the User-Agent header. This will - appear in parenthesis after the agent name. Custom dictates that this - should be a URL of a page explaining the purpose and behavior of this - crawler. - </description> -</property> - -<property> - <name>http.agent.email</name> - <value></value> - <description>An email address to advertise in the HTTP 'From' request - header and User-Agent header. A good practice is to mangle this - address (e.g. 'info at example dot com') to avoid spamming. - </description> -</property> - -<property> - <name>http.agent.version</name> - <value>Nutch-1.0-dev</value> - <description>A version string to advertise in the User-Agent - header.</description> -</property> - -<property> - <name>http.agent.host</name> - <value></value> - <description>Name or IP address of the host on which the Nutch crawler - would be running. Currently this is used by 'protocol-httpclient' - plugin. - </description> -</property> - -<property> - <name>http.timeout</name> - <value>10000</value> - <description>The default network timeout, in milliseconds.</description> -</property> - -<property> - <name>http.max.delays</name> - <value>100</value> - <description>The number of times a thread will delay when trying to - fetch a page. Each time it finds that a host is busy, it will wait - fetcher.server.delay. After http.max.delays attepts, it will give - up on the page for now.</description> -</property> - -<property> - <name>http.content.limit</name> - <value>65536</value> - <description>The length limit for downloaded content, in bytes. - If this value is nonnegative (>=0), content longer than it will be truncated; - otherwise, no truncation at all. - </description> -</property> - -<property> - <name>http.proxy.host</name> - <value></value> - <description>The proxy hostname. If empty, no proxy is used.</description> -</property> - -<property> - <name>http.proxy.port</name> - <value></value> - <description>The proxy port.</description> -</property> - -<property> - <name>http.proxy.username</name> - <value></value> - <description>Username for proxy. This will be used by - 'protocol-httpclient', if the proxy server requests basic, digest - and/or NTLM authentication. To use this, 'protocol-httpclient' must - be present in the value of 'plugin.includes' property. - NOTE: For NTLM authentication, do not prefix the username with the - domain, i.e. 'susam' is correct whereas 'DOMAIN\susam' is incorrect. - </description> -</property> - -<property> - <name>http.proxy.password</name> - <value></value> - <description>Password for proxy. This will be used by - 'protocol-httpclient', if the proxy server requests basic, digest - and/or NTLM authentication. To use this, 'protocol-httpclient' must - be present in the value of 'plugin.includes' property. - </description> -</property> - -<property> - <name>http.proxy.realm</name> - <value></value> - <description>Authentication realm for proxy. Do not define a value - if realm is not required or authentication should take place for any - realm. NTLM does not use the notion of realms. Specify the domain name - of NTLM authentication as the value for this property. To use this, - 'protocol-httpclient' must be present in the value of - 'plugin.includes' property. - </description> -</property> - -<property> - <name>http.auth.file</name> - <value>httpclient-auth.xml</value> - <description>Authentication configuration file for - 'protocol-httpclient' plugin. - </description> -</property> - -<property> - <name>http.verbose</name> - <value>false</value> - <description>If true, HTTP will log more verbosely.</description> -</property> - -<property> - <name>http.redirect.max</name> - <value>0</value> - <description>The maximum number of redirects the fetcher will follow when - trying to fetch a page. If set to negative or 0, fetcher won't immediately - follow redirected URLs, instead it will record them for later fetching. - </description> -</property> - -<property> - <name>http.useHttp11</name> - <value>false</value> - <description>NOTE: at the moment this works only for protocol-httpclient. - If true, use HTTP 1.1, if false use HTTP 1.0 . - </description> -</property> - -<!-- FTP properties --> - -<property> - <name>ftp.username</name> - <value>anonymous</value> - <description>ftp login username.</description> -</property> - -<property> - <name>ftp.password</name> - <value>anonymous@example.com</value> - <description>ftp login password.</description> -</property> - -<property> - <name>ftp.content.limit</name> - <value>65536</value> - <description>The length limit for downloaded content, in bytes. - If this value is nonnegative (>=0), content longer than it will be truncated; - otherwise, no truncation at all. - Caution: classical ftp RFCs never defines partial transfer and, in fact, - some ftp servers out there do not handle client side forced close-down very - well. Our implementation tries its best to handle such situations smoothly. - </description> -</property> - -<property> - <name>ftp.timeout</name> - <value>60000</value> - <description>Default timeout for ftp client socket, in millisec. - Please also see ftp.keep.connection below.</description> -</property> - -<property> - <name>ftp.server.timeout</name> - <value>100000</value> - <description>An estimation of ftp server idle time, in millisec. - Typically it is 120000 millisec for many ftp servers out there. - Better be conservative here. Together with ftp.timeout, it is used to - decide if we need to delete (annihilate) current ftp.client instance and - force to start another ftp.client instance anew. This is necessary because - a fetcher thread may not be able to obtain next request from queue in time - (due to idleness) before our ftp client times out or remote server - disconnects. Used only when ftp.keep.connection is true (please see below). - </description> -</property> - -<property> - <name>ftp.keep.connection</name> - <value>false</value> - <description>Whether to keep ftp connection. Useful if crawling same host - again and again. When set to true, it avoids connection, login and dir list - parser setup for subsequent urls. If it is set to true, however, you must - make sure (roughly): - (1) ftp.timeout is less than ftp.server.timeout - (2) ftp.timeout is larger than (fetcher.threads.fetch * fetcher.server.delay) - Otherwise there will be too many "delete client because idled too long" - messages in thread logs.</description> -</property> - -<property> - <name>ftp.follow.talk</name> - <value>false</value> - <description>Whether to log dialogue between our client and remote - server. Useful for debugging.</description> -</property> - -<!-- web db properties --> - -<property> - <name>db.default.fetch.interval</name> - <value>30</value> - <description>(DEPRECATED) The default number of days between re-fetches of a page. - </description> -</property> - -<property> - <name>db.fetch.interval.default</name> - <value>2592000</value> - <description>The default number of seconds between re-fetches of a page (30 days). - </description> -</property> - -<property> - <name>db.fetch.interval.max</name> - <value>7776000</value> - <description>The maximum number of seconds between re-fetches of a page - (90 days). After this period every page in the db will be re-tried, no - matter what is its status. - </description> -</property> - -<property> - <name>db.fetch.schedule.class</name> - <value>org.apache.nutch.crawl.DefaultFetchSchedule</value> - <description>The implementation of fetch schedule. DefaultFetchSchedule simply - adds the original fetchInterval to the last fetch time, regardless of - page changes.</description> -</property> - -<property> - <name>db.fetch.schedule.adaptive.inc_rate</name> - <value>0.4</value> - <description>If a page is unmodified, its fetchInterval will be - increased by this rate. This value should not - exceed 0.5, otherwise the algorithm becomes unstable.</description> -</property> - -<property> - <name>db.fetch.schedule.adaptive.dec_rate</name> - <value>0.2</value> - <description>If a page is modified, its fetchInterval will be - decreased by this rate. This value should not - exceed 0.5, otherwise the algorithm becomes unstable.</description> -</property> - -<property> - <name>db.fetch.schedule.adaptive.min_interval</name> - <value>60.0</value> - <description>Minimum fetchInterval, in seconds.</description> -</property> - -<property> - <name>db.fetch.schedule.adaptive.max_interval</name> - <value>31536000.0</value> - <description>Maximum fetchInterval, in seconds (365 days). - NOTE: this is limited by db.fetch.interval.max. Pages with - fetchInterval larger than db.fetch.interval.max - will be fetched anyway.</description> -</property> - -<property> - <name>db.fetch.schedule.adaptive.sync_delta</name> - <value>true</value> - <description>If true, try to synchronize with the time of page change. - by shifting the next fetchTime by a fraction (sync_rate) of the difference - between the last modification time, and the last fetch time.</description> -</property> - -<property> - <name>db.fetch.schedule.adaptive.sync_delta_rate</name> - <value>0.3</value> - <description>See sync_delta for description. This value should not - exceed 0.5, otherwise the algorithm becomes unstable.</description> -</property> - -<property> - <name>db.update.additions.allowed</name> - <value>true</value> - <description>If true, updatedb will add newly discovered URLs, if false - only already existing URLs in the CrawlDb will be updated and no new - URLs will be added. - </description> -</property> - -<property> - <name>db.ignore.internal.links</name> - <value>true</value> - <description>If true, when adding new links to a page, links from - the same host are ignored. This is an effective way to limit the - size of the link database, keeping only the highest quality - links. - </description> -</property> - -<property> - <name>db.ignore.external.links</name> - <value>false</value> - <description>If true, outlinks leading from a page to external hosts - will be ignored. This is an effective way to limit the crawl to include - only initially injected hosts, without creating complex URLFilters. - </description> -</property> - -<property> - <name>db.score.injected</name> - <value>1.0</value> - <description>The score of new pages added by the injector. - </description> -</property> - -<property> - <name>db.score.link.external</name> - <value>1.0</value> - <description>The score factor for new pages added due to a link from - another host relative to the referencing page's score. Scoring plugins - may use this value to affect initial scores of external links. - </description> -</property> - -<property> - <name>db.score.link.internal</name> - <value>1.0</value> - <description>The score factor for pages added due to a link from the - same host, relative to the referencing page's score. Scoring plugins - may use this value to affect initial scores of internal links. - </description> -</property> - -<property> - <name>db.score.count.filtered</name> - <value>false</value> - <description>The score value passed to newly discovered pages is - calculated as a fraction of the original page score divided by the - number of outlinks. If this option is false, only the outlinks that passed - URLFilters will count, if it's true then all outlinks will count. - </description> -</property> - -<property> - <name>db.max.inlinks</name> - <value>10000</value> - <description>Maximum number of Inlinks per URL to be kept in LinkDb. - If "invertlinks" finds more inlinks than this number, only the first - N inlinks will be stored, and the rest will be discarded. - </description> -</property> - -<property> - <name>db.max.outlinks.per.page</name> - <value>100</value> - <description>The maximum number of outlinks that we'll process for a page. - If this value is nonnegative (>=0), at most db.max.outlinks.per.page outlinks - will be processed for a page; otherwise, all outlinks will be processed. - </description> -</property> - -<property> - <name>db.max.anchor.length</name> - <value>100</value> - <description>The maximum number of characters permitted in an anchor. - </description> -</property> - -<property> - <name>db.fetch.retry.max</name> - <value>3</value> - <description>The maximum number of times a url that has encountered - recoverable errors is generated for fetch.</description> -</property> - -<property> - <name>db.signature.class</name> - <value>org.apache.nutch.crawl.MD5Signature</value> - <description>The default implementation of a page signature. Signatures - created with this implementation will be used for duplicate detection - and removal.</description> -</property> - -<property> - <name>db.signature.text_profile.min_token_len</name> - <value>2</value> - <description>Minimum token length to be included in the signature. - </description> -</property> - -<property> - <name>db.signature.text_profile.quant_rate</name> - <value>0.01</value> - <description>Profile frequencies will be rounded down to a multiple of - QUANT = (int)(QUANT_RATE * maxFreq), where maxFreq is a maximum token - frequency. If maxFreq > 1 then QUANT will be at least 2, which means that - for longer texts tokens with frequency 1 will always be discarded. - </description> -</property> - -<!-- generate properties --> - -<property> - <name>generate.max.per.host</name> - <value>-1</value> - <description>The maximum number of urls per host in a single - fetchlist. -1 if unlimited.</description> -</property> - -<property> - <name>generate.max.per.host.by.ip</name> - <value>false</value> - <description>If false, same host names are counted. If true, - hosts' IP addresses are resolved and the same IP-s are counted. - - -+-+-+- WARNING !!! -+-+-+- - When set to true, Generator will create a lot of DNS lookup - requests, rapidly. This may cause a DOS attack on - remote DNS servers, not to mention increased external traffic - and latency. For these reasons when using this option it is - required that a local caching DNS be used.</description> -</property> - -<property> - <name>generate.update.crawldb</name> - <value>false</value> - <description>For highly-concurrent environments, where several - generate/fetch/update cycles may overlap, setting this to true ensures - that generate will create different fetchlists even without intervening - updatedb-s, at the cost of running an additional job to update CrawlDB. - If false, running generate twice without intervening - updatedb will generate identical fetchlists.</description> -</property> - -<!-- fetcher properties --> - -<property> - <name>fetcher.server.delay</name> - <value>5.0</value> - <description>The number of seconds the fetcher will delay between - successive requests to the same server.</description> -</property> - -<property> - <name>fetcher.server.min.delay</name> - <value>0.0</value> - <description>The minimum number of seconds the fetcher will delay between - successive requests to the same server. This value is applicable ONLY - if fetcher.threads.per.host is greater than 1 (i.e. the host blocking - is turned off).</description> -</property> - -<property> - <name>fetcher.max.crawl.delay</name> - <value>30</value> - <description> - If the Crawl-Delay in robots.txt is set to greater than this value (in - seconds) then the fetcher will skip this page, generating an error report. - If set to -1 the fetcher will never skip such pages and will wait the - amount of time retrieved from robots.txt Crawl-Delay, however long that - might be. - </description> -</property> - -<property> - <name>fetcher.threads.fetch</name> - <value>10</value> - <description>The number of FetcherThreads the fetcher should use. - This is also determines the maximum number of requests that are - made at once (each FetcherThread handles one connection).</description> -</property> - -<property> - <name>fetcher.threads.per.host</name> - <value>1</value> - <description>This number is the maximum number of threads that - should be allowed to access a host at one time.</description> -</property> - -<property> - <name>fetcher.threads.per.host.by.ip</name> - <value>true</value> - <description>If true, then fetcher will count threads by IP address, - to which the URL's host name resolves. If false, only host name will be - used. NOTE: this should be set to the same value as - "generate.max.per.host.by.ip" - default settings are different only for - reasons of backward-compatibility.</description> -</property> - -<property> - <name>fetcher.verbose</name> - <value>false</value> - <description>If true, fetcher will log more verbosely.</description> -</property> - -<property> - <name>fetcher.parse</name> - <value>true</value> - <description>If true, fetcher will parse content.</description> -</property> - -<property> - <name>fetcher.store.content</name> - <value>true</value> - <description>If true, fetcher will store content.</description> -</property> - -<!-- indexer properties --> - -<property> - <name>indexer.score.power</name> - <value>0.5</value> - <description>Determines the power of link analyis scores. Each - pages's boost is set to <i>score<sup>scorePower</sup></i> where - <i>score</i> is its link analysis score and <i>scorePower</i> is the - value of this parameter. This is compiled into indexes, so, when - this is changed, pages must be re-indexed for it to take - effect.</description> -</property> - -<property> - <name>indexer.max.title.length</name> - <value>100</value> - <description>The maximum number of characters of a title that are indexed. - </description> -</property> - -<property> - <name>indexer.max.tokens</name> - <value>10000</value> - <description> - The maximum number of tokens that will be indexed for a single field - in a document. This limits the amount of memory required for - indexing, so that collections with very large files will not crash - the indexing process by running out of memory. - - Note that this effectively truncates large documents, excluding - from the index tokens that occur further in the document. If you - know your source documents are large, be sure to set this value - high enough to accomodate the expected size. If you set it to - -1, then the only limit is your memory, but you should anticipate - an OutOfMemoryError. - </description> -</property> - -<property> - <name>indexer.mergeFactor</name> - <value>50</value> - <description>The factor that determines the frequency of Lucene segment - merges. This must not be less than 2, higher values increase indexing - speed but lead to increased RAM usage, and increase the number of - open file handles (which may lead to "Too many open files" errors). - NOTE: the "segments" here have nothing to do with Nutch segments, they - are a low-level data unit used by Lucene. - </description> -</property> - -<property> - <name>indexer.minMergeDocs</name> - <value>50</value> - <description>This number determines the minimum number of Lucene - Documents buffered in memory between Lucene segment merges. Larger - values increase indexing speed and increase RAM usage. - </description> -</property> - -<property> - <name>indexer.maxMergeDocs</name> - <value>2147483647</value> - <description>This number determines the maximum number of Lucene - Documents to be merged into a new Lucene segment. Larger values - increase batch indexing speed and reduce the number of Lucene segments, - which reduces the number of open file handles; however, this also - decreases incremental indexing performance. - </description> -</property> - -<property> - <name>indexer.termIndexInterval</name> - <value>128</value> - <description>Determines the fraction of terms which Lucene keeps in - RAM when searching, to facilitate random-access. Smaller values use - more memory but make searches somewhat faster. Larger values use - less memory but make searches somewhat slower. - </description> -</property> - -<!-- indexingfilter plugin properties --> - -<property> - <name>indexingfilter.order</name> - <value></value> - <description>The order by which index filters are applied. - If empty, all available index filters (as dictated by properties - plugin-includes and plugin-excludes above) are loaded and applied in system - defined order. If not empty, only named filters are loaded and applied - in given order. For example, if this property has value: - org.apache.nutch.indexer.basic.BasicIndexingFilter org.apache.nutch.indexer.more.MoreIndexingFilter - then BasicIndexingFilter is applied first, and MoreIndexingFilter second. - - Filter ordering might have impact on result if one filter depends on output of - another filter. - </description> -</property> - - -<!-- analysis properties --> - -<property> - <name>analysis.common.terms.file</name> - <value>common-terms.utf8</value> - <description>The name of a file containing a list of common terms - that should be indexed in n-grams.</description> -</property> - -<!-- searcher properties --> - -<property> - <name>searcher.dir</name> - <value>crawl</value> - <description> - Path to root of crawl. This directory is searched (in - order) for either the file search-servers.txt, containing a list of - distributed search servers, or the directory "index" containing - merged indexes, or the directory "segments" containing segment - indexes. - </description> -</property> - -<property> - <name>searcher.filter.cache.size</name> - <value>16</value> - <description> - Maximum number of filters to cache. Filters can accelerate certain - field-based queries, like language, document format, etc. Each - filter requires one bit of RAM per page. So, with a 10 million page - index, a cache size of 16 consumes two bytes per page, or 20MB. - </description> -</property> - -<property> - <name>searcher.filter.cache.threshold</name> - <value>0.05</value> - <description> - Filters are cached when their term is matched by more than this - fraction of pages. For example, with a threshold of 0.05, and 10 - million pages, the term must match more than 1/20, or 50,000 pages. - So, if out of 10 million pages, 50% of pages are in English, and 2% - are in Finnish, then, with a threshold of 0.05, searches for - "lang:en" will use a cached filter, while searches for "lang:fi" - will score all 20,000 finnish documents. - </description> -</property> - -<property> - <name>searcher.hostgrouping.rawhits.factor</name> - <value>2.0</value> - <description> - A factor that is used to determine the number of raw hits - initially fetched, before host grouping is done. - </description> -</property> - -<property> - <name>searcher.summary.context</name> - <value>5</value> - <description> - The number of context terms to display preceding and following - matching terms in a hit summary. - </description> -</property> - -<property> - <name>searcher.summary.length</name> - <value>20</value> - <description> - The total number of terms to display in a hit summary. - </description> -</property> - -<property> - <name>searcher.max.hits</name> - <value>-1</value> - <description>If positive, search stops after this many hits are - found. Setting this to small, positive values (e.g., 1000) can make - searches much faster. With a sorted index, the quality of the hits - suffers little.</description> -</property> - -<property> - <name>searcher.max.time.tick_count</name> - <value>-1</value> - <description>If positive value is defined here, limit search time for - every request to this number of elapsed ticks (see the tick_length - property below). The total maximum time for any search request will be - then limited to tick_count * tick_length milliseconds. When search time - is exceeded, partial results will be returned, and the total number of - hits will be estimated. - </description> -</property> - -<property> - <name>searcher.max.time.tick_length</name> - <value>200</value> - <description>The number of milliseconds between ticks. Larger values - reduce the timer granularity (precision). Smaller values bring more - overhead. - </description> -</property> - -<property> - <name>searcher.num.handlers</name> - <value>10</value> - <description>The number of handlers for the distributed search server. - </description> -</property> - -<property> - <name>searcher.max.hits.per.page</name> - <value>1000</value> - <description> The maximum number of hits to show per page. -1 if - unlimited. If the number of hits requested by the user (via - hitsPerPage parameter in the query string) is more than the value - specified in this property, then this value is assumed as the number - of hits per page. - </description> -</property> - -<!-- URL normalizer properties --> - -<property> - <name>urlnormalizer.order</name> - <value>org.apache.nutch.net.urlnormalizer.basic.BasicURLNormalizer org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer</value> - <description>Order in which normalizers will run. If any of these isn't - activated it will be silently skipped. If other normalizers not on the - list are activated, they will run in random order after the ones - specified here are run. - </description> -</property> - -<property> - <name>urlnormalizer.regex.file</name> - <value>regex-normalize.xml</value> - <description>Name of the config file used by the RegexUrlNormalizer class. - </description> -</property> - -<property> - <name>urlnormalizer.loop.count</name> - <value>1</value> - <description>Optionally loop through normalizers several times, to make - sure that all transformations have been performed. - </description> -</property> - -<!-- mime properties --> - -<property> - <name>mime.types.file</name> - <value>tika-mimetypes.xml</value> - <description>Name of file in CLASSPATH containing filename extension and - magic sequence to mime types mapping information</description> -</property> - -<property> - <name>mime.type.magic</name> - <value>true</value> - <description>Defines if the mime content type detector uses magic resolution. - </description> -</property> - -<!-- plugin properties --> - -<property> - <name>plugin.folders</name> - <value>plugins</value> - <description>Directories where nutch plugins are located. Each - element may be a relative or absolute path. If absolute, it is used - as is. If relative, it is searched for on the classpath.</description> -</property> - -<property> - <name>plugin.auto-activation</name> - <value>true</value> - <description>Defines if some plugins that are not activated regarding - the plugin.includes and plugin.excludes properties must be automaticaly - activated if they are needed by some actived plugins. - </description> -</property> - -<property> - <name>plugin.includes</name> - <value>protocol-http|urlfilter-regex|parse-(text|html|js)|index-(basic|anchor)|query-(basic|site|url)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic)</value> - <description>Regular expression naming plugin directory names to - include. Any plugin not matching this expression is excluded. - In any case you need at least include the nutch-extensionpoints plugin. By - default Nutch includes crawling just HTML and plain text via HTTP, - and basic indexing and search plugins. In order to use HTTPS please enable - protocol-httpclient, but be aware of possible intermittent problems with the - underlying commons-httpclient library. - </description> -</property> - -<property> - <name>plugin.excludes</name> - <value></value> - <description>Regular expression naming plugin directory names to exclude. - </description> -</property> - -<!-- parser properties --> - -<property> - <name>parse.plugin.file</name> - <value>parse-plugins.xml</value> - <description>The name of the file that defines the associations between - content-types and parsers.</description> -</property> - -<property> - <name>parser.character.encoding.default</name> - <value>windows-1252</value> - <description>The character encoding to fall back to when no other information - is available</description> -</property> - -<property> - <name>encodingdetector.charset.min.confidence</name> - <value>-1</value> - <description>A integer between 0-100 indicating minimum confidence value - for charset auto-detection. Any negative value disables auto-detection. - </description> -</property> - -<property> - <name>parser.caching.forbidden.policy</name> - <value>content</value> - <description>If a site (or a page) requests through its robot metatags - that it should not be shown as cached content, apply this policy. Currently - three keywords are recognized: "none" ignores any "noarchive" directives. - "content" doesn't show the content, but shows summaries (snippets). - "all" doesn't show either content or summaries.</description> -</property> - - -<property> - <name>parser.html.impl</name> - <value>neko</value> - <description>HTML Parser implementation. Currently the following keywords - are recognized: "neko" uses NekoHTML, "tagsoup" uses TagSoup. - </description> -</property> - -<property> - <name>parser.html.form.use_action</name> - <value>false</value> - <description>If true, HTML parser will collect URLs from form action - attributes. This may lead to undesirable behavior (submitting empty - forms during next fetch cycle). If false, form action attribute will - be ignored.</description> -</property> - -<property> - <name>parser.html.outlinks.ignore_tags</name> - <value></value> - <description>Comma separated list of HTML tags, from which outlinks - shouldn't be extracted. Nutch takes links from: a, area, form, frame, - iframe, script, link, img. If you add any of those tags here, it - won't be taken. Default is empty list. Probably reasonable value - for most people would be "img,script,link".</description> -</property> - - -<!-- urlfilter plugin properties --> - -<property> - <name>urlfilter.regex.file</name> - <value>regex-urlfilter.txt</value> - <description>Name of file on CLASSPATH containing regular expressions - used by urlfilter-regex (RegexURLFilter) plugin.</description> -</property> - -<property> - <name>urlfilter.automaton.file</name> - <value>automaton-urlfilter.txt</value> - <description>Name of file on CLASSPATH containing regular expressions - used by urlfilter-automaton (AutomatonURLFilter) plugin.</description> -</property> - -<property> - <name>urlfilter.prefix.file</name> - <value>prefix-urlfilter.txt</value> - <description>Name of file on CLASSPATH containing url prefixes - used by urlfilter-prefix (PrefixURLFilter) plugin.</description> -</property> - -<property> - <name>urlfilter.suffix.file</name> - <value>suffix-urlfilter.txt</value> - <description>Name of file on CLASSPATH containing url suffixes - used by urlfilter-suffix (SuffixURLFilter) plugin.</description> -</property> - -<property> - <name>urlfilter.order</name> - <value></value> - <description>The order by which url filters are applied. - If empty, all available url filters (as dictated by properties - plugin-includes and plugin-excludes above) are loaded and applied in system - defined order. If not empty, only named filters are loaded and applied - in given order. For example, if this property has value: - org.apache.nutch.urlfilter.regex.RegexURLFilter org.apache.nutch.urlfilter.prefix.PrefixURLFilter - then RegexURLFilter is applied first, and PrefixURLFilter second. - Since all filters are AND'ed, filter ordering does not have impact - on end result, but it may have performance implication, depending - on relative expensiveness of filters. - </description> -</property> - -<!-- scoring filters properties --> - -<property> - <name>scoring.filter.order</name> - <value></value> - <description>The order in which scoring filters are applied. - This may be left empty (in which case all available scoring - filters will be applied in the order defined in plugin-includes - and plugin-excludes), or a space separated list of implementation - classes. - </description> -</property> - -<!-- clustering extension properties --> - -<property> - <name>extension.clustering.hits-to-cluster</name> - <value>100</value> - <description>Number of snippets retrieved for the clustering extension - if clustering extension is available and user requested results - to be clustered.</description> -</property> - -<property> - <name>extension.clustering.extension-name</name> - <value></value> - <description>Use the specified online clustering extension. If empty, - the first available extension will be used. The "name" here refers to an 'id' - attribute of the 'implementation' element in the plugin descriptor XML - file.</description> -</property> - -<!-- ontology extension properties --> - -<property> - <name>extension.ontology.extension-name</name> - <value></value> - <description>Use the specified online ontology extension. If empty, - the first available extension will be used. The "name" here refers to an 'id' - attribute of the 'implementation' element in the plugin descriptor XML - file.</description> -</property> - -<property> - <name>extension.ontology.urls</name> - <value> - </value> - <description>Urls of owl files, separated by spaces, such as - http://www.example.com/ontology/time.owl - http://www.example.com/ontology/space.owl - http://www.example.com/ontology/wine.owl - Or - file:/ontology/time.owl - file:/ontology/space.owl - file:/ontology/wine.owl - You have to make sure each url is valid. - By default, there is no owl file, so query refinement based on ontology - is silently ignored. - </description> -</property> - -<!-- query-basic plugin properties --> - -<property> - <name>query.url.boost</name> - <value>4.0</value> - <description> Used as a boost for url field in Lucene query. - </description> -</property> - -<property> - <name>query.anchor.boost</name> - <value>2.0</value> - <description> Used as a boost for anchor field in Lucene query. - </description> -</property> - -<property> - <name>query.title.boost</name> - <value>1.5</value> - <description> Used as a boost for title field in Lucene query. - </description> -</property> - -<property> - <name>query.host.boost</name> - <value>2.0</value> - <description> Used as a boost for host field in Lucene query. - </description> -</property> - -<property> - <name>query.phrase.boost</name> - <value>1.0</value> - <description> Used as a boost for phrase in Lucene query. - Multiplied by boost for field phrase is matched in. - </description> -</property> - -<!-- creative-commons plugin properties --> - -<property> - <name>query.cc.boost</name> - <value>0.0</value> - <description> Used as a boost for cc field in Lucene query. - </description> -</property> - -<!-- query-more plugin properties --> - -<property> - <name>query.type.boost</name> - <value>0.0</value> - <description> Used as a boost for type field in Lucene query. - </description> -</property> - -<!-- query-site plugin properties --> - -<property> - <name>query.site.boost</name> - <value>0.0</value> - <description> Used as a boost for site field in Lucene query. - </description> -</property> - -<!-- microformats-reltag plugin properties --> - -<property> - <name>query.tag.boost</name> - <value>1.0</value> - <description> Used as a boost for tag field in Lucene query. - </description> -</property> - -<!-- language-identifier plugin properties --> - -<property> - <name>lang.ngram.min.length</name> - <value>1</value> - <description> The minimum size of ngrams to uses to identify - language (must be between 1 and lang.ngram.max.length). - The larger is the range between lang.ngram.min.length and - lang.ngram.max.length, the better is the identification, but - the slowest it is. - </description> -</property> - -<property> - <name>lang.ngram.max.length</name> - <value>4</value> - <description> The maximum size of ngrams to uses to identify - language (must be between lang.ngram.min.length and 4). - The larger is the range between lang.ngram.min.length and - lang.ngram.max.length, the better is the identification, but - the slowest it is. - </description> -</property> - -<property> - <name>lang.analyze.max.length</name> - <value>2048</value> - <description> The maximum bytes of data to uses to indentify - the language (0 means full content analysis). - The larger is this value, the better is the analysis, but the - slowest it is. - </description> -</property> - -<property> - <name>query.lang.boost</name> - <value>0.0</value> - <description> Used as a boost for lang field in Lucene query. - </description> -</property> - -<!-- Temporary Hadoop 0.17.x workaround. --> - -<property> - <name>hadoop.job.history.user.location</name> - <value>${hadoop.log.dir}/history/user</value> - <description>Hadoop 0.17.x comes with a default setting to create - user logs inside the output path of the job. This breaks some - Hadoop classes, which expect the output to contain only - part-XXXXX files. This setting changes the output to a - subdirectory of the regular log directory. - </description> -</property> - -</configuration> diff --git a/search-server/nutch/crawl_jsp/conf/nutch-site.xml b/search-server/nutch/crawl_jsp/conf/nutch-site.xml deleted file mode 100644 index 70975c2..0000000 --- a/search-server/nutch/crawl_jsp/conf/nutch-site.xml +++ /dev/null @@ -1,58 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<!-- Put site-specific property overrides in this file. --> - -<configuration> - -<property> - <name>http.agent.name</name> - <value>Spacewalk</value> - <description>HTTP 'User-Agent' request header. MUST NOT be empty - - please set this to a single word uniquely related to your organization. - - NOTE: You should also check other related properties: - - http.robots.agents - http.agent.description - http.agent.url - http.agent.email - http.agent.version - - and set their values appropriately. - - </description> -</property> - -<property> - <name>fetcher.server.delay</name> - <value>0</value> - <description>The number of seconds the fetcher will delay between - successive requests to the same server.</description> -</property> - -<property> - <name>fetcher.threads.per.host</name> - <value>50</value> - <description>This number is the maximum number of threads that - should be allowed to access a host at one time.</description> -</property> - -<property> - <name>plugin.folders</name> - <value>/usr/share/nutch/plugins</value> - <description>Directories where nutch plugins are located. Each - element may be a relative or absolute path. If absolute, it is used - as is. If relative, it is searched for on the classpath.</description> -</property> - -<property> -<name>plugin.includes</name> - <value>protocol-file|urlfilter-regex|parse-(xml|text|html|js|pdf)|index-basic|query-(basic|site|url)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic)</value> -</property> - -<property> - <name>file.content.limit</name> <value>-1</value> -</property> - -</configuration> diff --git a/search-server/nutch/crawl_jsp/conf/parse-plugins.dtd b/search-server/nutch/crawl_jsp/conf/parse-plugins.dtd deleted file mode 100644 index ae21045..0000000 --- a/search-server/nutch/crawl_jsp/conf/parse-plugins.dtd +++ /dev/null @@ -1,12 +0,0 @@ -<!ELEMENT parse-plugins (mimeType+,aliases)> -<!ELEMENT mimeType (plugin+)> -<!ATTLIST mimeType name CDATA #REQUIRED> - -<!ELEMENT plugin EMPTY> -<!ATTLIST plugin id CDATA #REQUIRED> -<!ATTLIST plugin order CDATA ''> - -<!ELEMENT aliases (alias+)> -<!ELEMENT alias EMPTY> -<!ATTLIST alias name CDATA #REQUIRED> -<!ATTLIST alias extension-id CDATA #REQUIRED> \ No newline at end of file diff --git a/search-server/nutch/crawl_jsp/conf/parse-plugins.xml b/search-server/nutch/crawl_jsp/conf/parse-plugins.xml deleted file mode 100644 index 876d188..0000000 --- a/search-server/nutch/crawl_jsp/conf/parse-plugins.xml +++ /dev/null @@ -1,221 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Author : mattmann - Description: This xml file represents a natural ordering for which parsing - plugin should get called for a particular mimeType. ---> - -<parse-plugins> - - <mimeType name="application/msword"> - <plugin id="parse-msword" /> - </mimeType> - - <mimeType name="application/pdf"> - <plugin id="parse-pdf" /> - </mimeType> - - <mimeType name="application/postscript"> - <plugin id="parse-pdf" /> - </mimeType> - - <mimeType name="application/rss+xml"> - <plugin id="parse-rss" /> - <plugin id="feed" /> - </mimeType> - - <mimeType name="application/vnd.ms-excel"> - <plugin id="parse-msexcel" /> - </mimeType> - - <mimeType name="application/vnd.ms-powerpoint"> - <plugin id="parse-mspowerpoint" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.text"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.text-template"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.text-master"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.text-web"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.presentation"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.presentation-template"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.spreadsheet"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.oasis.opendocument.spreadsheet-template"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.sun.xml.calc"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.sun.xml.calc.template"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.sun.xml.impress"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.sun.xml.impress.template"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.sun.xml.writer"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/vnd.sun.xml.writer.template"> - <plugin id="parse-oo" /> - </mimeType> - - <mimeType name="application/xhtml+xml"> - <plugin id="parse-html" /> - </mimeType> - - <mimeType name="application/x-bzip2"> - <!-- try and parse it with the zip parser --> - <plugin id="parse-zip" /> - </mimeType> - - <mimeType name="application/x-csh"> - <plugin id="parse-text" /> - </mimeType> - - <mimeType name="application/x-gzip"> - <!-- try and parse it with the zip parser --> - <plugin id="parse-zip" /> - </mimeType> - - <mimeType name="application/x-javascript"> - <plugin id="parse-js" /> - </mimeType> - - <mimeType name="application/x-kword"> - <!-- try and parse it with the word parser --> - <plugin id="parse-msword" /> - </mimeType> - - <mimeType name="application/x-kspread"> - <!-- try and parse it with the msexcel parser --> - <plugin id="parse-msexcel" /> - </mimeType> - - <mimeType name="application/x-shockwave-flash"> - <plugin id="parse-swf" /> - </mimeType> - - <mimeType name="application/zip"> - <plugin id="parse-zip" /> - </mimeType> - - <mimeType name="text/html"> - <plugin id="parse-html" /> - </mimeType> - - <mimeType name="text/plain"> - <plugin id="parse-text" /> - </mimeType> - - <mimeType name="text/richtext"> - <plugin id="parse-rtf" /> - <plugin id="parse-msword" /> - </mimeType> - - <mimeType name="text/rtf"> - <plugin id="parse-rtf" /> - <plugin id="parse-msword" /> - </mimeType> - - <mimeType name="text/sgml"> - <plugin id="parse-html" /> - </mimeType> - - <mimeType name="text/tab-separated-values"> - <plugin id="parse-msexcel" /> - </mimeType> - - <mimeType name="text/xml"> - <plugin id="parse-html" /> - <plugin id="parse-rss" /> - <plugin id="feed" /> - </mimeType> - - <!-- Types for parse-ext plugin: required for unit tests to pass. --> - - <mimeType name="application/vnd.nutch.example.cat"> - <plugin id="parse-ext" /> - </mimeType> - - <mimeType name="application/vnd.nutch.example.md5sum"> - <plugin id="parse-ext" /> - </mimeType> - - <!-- alias mappings for parse-xxx names to the actual extension implementation - ids described in each plugin's plugin.xml file --> - <aliases> - <alias name="parse-ext" extension-id="ExtParser" /> - <alias name="parse-html" - extension-id="org.apache.nutch.parse.html.HtmlParser" /> - <alias name="parse-js" extension-id="JSParser" /> - <alias name="parse-mp3" - extension-id="org.apache.nutch.parse.mp3.MP3Parser" /> - <alias name="parse-msexcel" - extension-id="org.apache.nutch.parse.msexcel.MSExcelParser" /> - <alias name="parse-mspowerpoint" - extension-id="org.apache.nutch.parse.mspowerpoint.MSPowerPointParser" /> - <alias name="parse-msword" - extension-id="org.apache.nutch.parse.msword.MSWordParser" /> - <alias name="parse-oo" - extension-id="org.apache.nutch.parse.oo.OpenDocument.Text" /> - <alias name="parse-pdf" - extension-id="org.apache.nutch.parse.pdf.PdfParser" /> - <alias name="parse-rss" - extension-id="org.apache.nutch.parse.rss.RSSParser" /> - <alias name="feed" - extension-id="org.apache.nutch.parse.feed.FeedParser" /> - <alias name="parse-rtf" - extension-id="org.apache.nutch.parse.rtf.RTFParseFactory" /> - <alias name="parse-swf" - extension-id="org.apache.nutch.parse.swf.SWFParser" /> - <alias name="parse-text" - extension-id="org.apache.nutch.parse.text.TextParser" /> - <alias name="parse-zip" - extension-id="org.apache.nutch.parse.zip.ZipParser" /> - </aliases> - -</parse-plugins> diff --git a/search-server/nutch/crawl_jsp/conf/prefix-urlfilter.txt b/search-server/nutch/crawl_jsp/conf/prefix-urlfilter.txt deleted file mode 100644 index 491dc1f..0000000 --- a/search-server/nutch/crawl_jsp/conf/prefix-urlfilter.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# config file for urlfilter-prefix plugin - -http:// -https:// -ftp:// -file:// diff --git a/search-server/nutch/crawl_jsp/conf/regex-normalize.xml b/search-server/nutch/crawl_jsp/conf/regex-normalize.xml deleted file mode 100644 index 8d8bad1..0000000 --- a/search-server/nutch/crawl_jsp/conf/regex-normalize.xml +++ /dev/null @@ -1,66 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<!-- This is the configuration file for the RegexUrlNormalize Class. - This is intended so that users can specify substitutions to be - done on URLs. The regex engine that is used is Perl5 compatible. - The rules are applied to URLs in the order they occur in this file. --> - -<!-- WATCH OUT: an xml parser reads this file an ampersands must be - expanded to & --> - -<!-- The following rules show how to strip out session IDs, default pages, - interpage anchors, etc. Order does matter! --> -<regex-normalize> - -<!-- removes session ids from urls (such as jsessionid and PHPSESSID) --> -<regex> - <pattern>([;_]?((?i)l|j|bv_)?((?i)sid|phpsessid|sessionid)=.*?)(?|&|#|$)</pattern> - <substitution>$4</substitution> -</regex> - -<!-- changes default pages into standard for /index.html, etc. into / -<regex> - <pattern>/((?i)index|default)\.((?i)js[pf]{1}?[afx]?|cgi|cfm|asp[x]?|[psx]?htm[l]?|php[3456]?)(\?|&|#|$)</pattern> - <substitution>/$3</substitution> -</regex> --> - -<!-- removes interpage href anchors such as site.com#location --> -<regex> - <pattern>#.*?(?|&|$)</pattern> - <substitution>$1</substitution> -</regex> - -<!-- cleans ?&var=value into ?var=value --> -<regex> - <pattern>?&</pattern> - <substitution>?</substitution> -</regex> - -<!-- cleans multiple sequential ampersands into a single ampersand --> -<regex> - <pattern>&{2,}</pattern> - <substitution>&</substitution> -</regex> - -<!-- removes trailing ? --> -<regex> - <pattern>[?&.]$</pattern> - <substitution></substitution> -</regex> - -</regex-normalize> diff --git a/search-server/nutch/crawl_jsp/conf/regex-urlfilter.txt b/search-server/nutch/crawl_jsp/conf/regex-urlfilter.txt deleted file mode 100644 index fe56603..0000000 --- a/search-server/nutch/crawl_jsp/conf/regex-urlfilter.txt +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# The default url filter. -# Better for whole-internet crawling. - -# Each non-comment, non-blank line contains a regular expression -# prefixed by '+' or '-'. The first matching pattern in the file -# determines whether a URL is included or ignored. If no pattern -# matches, the URL is ignored. - -# skip file: ftp: and mailto: urls --^(file|ftp|mailto): - -# skip image and other suffixes we can't yet parse --.(gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|jpeg|JPEG|bmp|BMP)$ - -# skip URLs containing certain characters as probable queries, etc. --[?*!@=] - -# skip URLs with slash-delimited segment that repeats 3+ times, to break loops --.*(/[^/]+)/[^/]+\1/[^/]+\1/ - -# accept anything else -+. diff --git a/search-server/nutch/crawl_jsp/conf/slaves b/search-server/nutch/crawl_jsp/conf/slaves deleted file mode 100644 index 2fbb50c..0000000 --- a/search-server/nutch/crawl_jsp/conf/slaves +++ /dev/null @@ -1 +0,0 @@ -localhost diff --git a/search-server/nutch/crawl_jsp/conf/subcollections.xml b/search-server/nutch/crawl_jsp/conf/subcollections.xml deleted file mode 100644 index 7b8805d..0000000 --- a/search-server/nutch/crawl_jsp/conf/subcollections.xml +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<subcollections> - <subcollection> - <name>nutch</name> - <id>nutch</id> - <whitelist> -http://lucene.apache.org/nutch/ -http://wiki.apache.org/nutch/ - </whitelist> - <blacklist /> - </subcollection> -</subcollections> diff --git a/search-server/nutch/crawl_jsp/conf/suffix-urlfilter.txt b/search-server/nutch/crawl_jsp/conf/suffix-urlfilter.txt deleted file mode 100644 index b510d4a..0000000 --- a/search-server/nutch/crawl_jsp/conf/suffix-urlfilter.txt +++ /dev/null @@ -1,91 +0,0 @@ -# config file for urlfilter-suffix plugin - -# case-insensitive, allow unknown suffixes -+I -# uncomment the line below to filter on url path -#+P - -### prohibit these -# pictures -.gif -.jpg -.jpeg -.bmp -.png -.tif -.tiff -.ico -.eps -.ps -.wmf -.fpx -.cur -.ani -.img -.lwf -.pcd -.psp -.psd -.tga -.xbm -.xpm - -# web-formats -.css - -# archives/packages -.arj -.arc -.7z -.cab -.lzw -.lha -.lzh -.zip -.gz -.tar -.tgz -.sit -.rpm -.deb -.pkg - -# audio/video -.mid -.midi -.rmi -.mpeg -.mpg -.mpe -.mp3 -.mp2 -.aac -.mov -.fla -.flv -.ra -.ram -.rm -.rmv -.wma -.wmv -.wav -.wave -.ogg -.avi -.au -.snd - -# executables -.exe -.com - -# windows links -.lnk - -# typo3-extensions -.t3x - -# disc-images -.iso -.bin diff --git a/search-server/nutch/crawl_jsp/conf/tika-mimetypes.xml b/search-server/nutch/crawl_jsp/conf/tika-mimetypes.xml deleted file mode 100644 index ce84078..0000000 --- a/search-server/nutch/crawl_jsp/conf/tika-mimetypes.xml +++ /dev/null @@ -1,367 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Description: This xml file defines the valid mime types used by Tika. - The mime types within this file are based on the types in the mime-types.xml - file available in Apache Nutch. ---> - -<mime-info> - - <mime-type type="text/plain"> - <magic priority="50"> - <match value="This is TeX," type="string" offset="0" /> - <match value="This is METAFONT," type="string" offset="0" /> - </magic> - <glob pattern="*.txt" /> - <glob pattern="*.asc" /> - </mime-type> - - <mime-type type="text/html"> - <magic priority="50"> - <match value="<!DOCTYPE HTML" type="string" - offset="0:64" /> - <match value="<!doctype html" type="string" - offset="0:64" /> - <match value="<HEAD" type="string" offset="0:64" /> - <match value="<head" type="string" offset="0:64" /> - <match value="<TITLE" type="string" offset="0:64" /> - <match value="<title" type="string" offset="0:64" /> - <match value="<html" type="string" offset="0:64" /> - <match value="<HTML" type="string" offset="0:64" /> - <match value="<BODY" type="string" offset="0" /> - <match value="<body" type="string" offset="0" /> - <match value="<TITLE" type="string" offset="0" /> - <match value="<title" type="string" offset="0" /> - <match value="<!--" type="string" offset="0" /> - <match value="<h1" type="string" offset="0" /> - <match value="<H1" type="string" offset="0" /> - <match value="<!doctype HTML" type="string" offset="0" /> - <match value="<!DOCTYPE html" type="string" offset="0" /> - </magic> - <glob pattern="*.html" /> - <glob pattern="*.htm" /> - </mime-type> - - <mime-type type="application/xhtml+xml"> - <glob pattern="*.xhtml" /> - <root-XML namespaceURI='http://www.w3.org/1999/xhtml' - localName='html' /> - </mime-type> - - <mime-type type="application/vnd.ms-powerpoint"> - <glob pattern="*.ppz" /> - <glob pattern="*.ppt" /> - <glob pattern="*.pps" /> - <glob pattern="*.pot" /> - <magic priority="50"> - <match value="0xcfd0e011" type="little32" offset="0" /> - </magic> - </mime-type> - - <mime-type type="application/vnd.ms-excel"> - <magic priority="50"> - <match value="Microsoft Excel 5.0 Worksheet" type="string" - offset="2080" /> - </magic> - <glob pattern="*.xls" /> - <glob pattern="*.xlc" /> - <glob pattern="*.xll" /> - <glob pattern="*.xlm" /> - <glob pattern="*.xlw" /> - <glob pattern="*.xla" /> - <glob pattern="*.xlt" /> - <glob pattern="*.xld" /> - <alias type="application/msexcel" /> - </mime-type> - - <mime-type type="application/vnd.oasis.opendocument.text"> - <glob pattern="*.odt" /> - </mime-type> - - - <mime-type type="application/zip"> - <alias type="application/x-zip-compressed" /> - <magic priority="40"> - <match value="PK\003\004" type="string" offset="0" /> - </magic> - <glob pattern="*.zip" /> - </mime-type> - - <mime-type type="application/vnd.oasis.opendocument.text"> - <glob pattern="*.oth" /> - </mime-type> - - <mime-type type="application/msword"> - <magic priority="50"> - <match value="\x31\xbe\x00\x00" type="string" offset="0" /> - <match value="PO^Q`" type="string" offset="0" /> - <match value="\376\067\0\043" type="string" offset="0" /> - <match value="\333\245-\0\0\0" type="string" offset="0" /> - <match value="Microsoft Word 6.0 Document" type="string" - offset="2080" /> - <match value="Microsoft Word document data" type="string" - offset="2112" /> - </magic> - <glob pattern="*.doc" /> - <alias type="application/vnd.ms-word" /> - </mime-type> - - <mime-type type="application/octet-stream"> - <magic priority="50"> - <match value="\037\036" type="string" offset="0" /> - <match value="017437" type="host16" offset="0" /> - <match value="0x1fff" type="host16" offset="0" /> - <match value="\377\037" type="string" offset="0" /> - <match value="0145405" type="host16" offset="0" /> - </magic> - <glob pattern="*.bin" /> - </mime-type> - - <mime-type type="application/pdf"> - <magic priority="50"> - <match value="%PDF-" type="string" offset="0" /> - </magic> - <glob pattern="*.pdf" /> - <alias type="application/x-pdf" /> - </mime-type> - - <mime-type type="application/atom+xml"> - <root-XML localName="feed" - namespaceURI="http://purl.org/atom/ns#" /> - </mime-type> - - <mime-type type="application/mac-binhex40"> - <glob pattern="*.hqx" /> - </mime-type> - - <mime-type type="application/mac-compactpro"> - <glob pattern="*.cpt" /> - </mime-type> - - <mime-type type="application/rtf"> - <glob pattern="*.rtf"/> - <alias type="text/rtf" /> - </mime-type> - - <mime-type type="application/rss+xml"> - <alias type="text/rss" /> - <root-XML localName="rss" /> - <root-XML namespaceURI="http://purl.org/rss/1.0/" /> - <glob pattern="*.rss" /> - </mime-type> - - <!-- added in by mattmann --> - <mime-type type="application/xml"> - <alias type="text/xml" /> - <glob pattern="*.xml" /> - </mime-type> - - <mime-type type="application/x-mif"> - <alias type="application/vnd.mif" /> - </mime-type> - - <mime-type type="application/vnd.wap.wbxml"> - <glob pattern="*.wbxml" /> - </mime-type> - - <mime-type type="application/vnd.wap.wmlc"> - <_comment>Compiled WML Document</_comment> - <glob pattern="*.wmlc" /> - </mime-type> - - <mime-type type="application/vnd.wap.wmlscriptc"> - <_comment>Compiled WML Script</_comment> - <glob pattern="*.wmlsc" /> - </mime-type> - - <mime-type type="text/vnd.wap.wmlscript"> - <_comment>WML Script</_comment> - <glob pattern="*.wmls" /> - </mime-type> - - <mime-type type="application/x-bzip"> - <alias type="application/x-bzip2" /> - </mime-type> - - <mime-type type="application/x-bzip-compressed-tar"> - <glob pattern="*.tbz" /> - <glob pattern="*.tbz2" /> - </mime-type> - - <mime-type type="application/x-cdlink"> - <_comment>Virtual CD-ROM CD Image File</_comment> - <glob pattern="*.vcd" /> - </mime-type> - - <mime-type type="application/x-director"> - <_comment>Shockwave Movie</_comment> - <glob pattern="*.dcr" /> - <glob pattern="*.dir" /> - <glob pattern="*.dxr" /> - </mime-type> - - <mime-type type="application/x-futuresplash"> - <_comment>Macromedia FutureSplash File</_comment> - <glob pattern="*.spl" /> - </mime-type> - - <mime-type type="application/x-java"> - <alias type="application/java" /> - </mime-type> - - <mime-type type="application/x-koan"> - <_comment>SSEYO Koan File</_comment> - <glob pattern="*.skp" /> - <glob pattern="*.skd" /> - <glob pattern="*.skt" /> - <glob pattern="*.skm" /> - </mime-type> - - <mime-type type="application/x-latex"> - <_comment>LaTeX Source Document</_comment> - <glob pattern="*.latex" /> - </mime-type> - - <!-- JC CHANGED - <mime-type type="application/x-mif"> - <_comment>FrameMaker MIF document</_comment> - <glob pattern="*.mif"/> - </mime-type> --> - - <mime-type type="application/x-ms-dos-executable"> - <alias type="application/x-dosexec" /> - </mime-type> - - <mime-type type="application/ogg"> - <alias type="application/x-ogg" /> - </mime-type> - - <mime-type type="application/x-rar"> - <alias type="application/x-rar-compressed" /> - </mime-type> - - <mime-type type="application/x-shellscript"> - <alias type="application/x-sh" /> - </mime-type> - - <mime-type type="application/xhtml+xml"> - <glob pattern="*.xht" /> - </mime-type> - - <mime-type type="audio/midi"> - <glob pattern="*.kar" /> - </mime-type> - - <mime-type type="audio/x-pn-realaudio"> - <alias type="audio/x-realaudio" /> - </mime-type> - - <mime-type type="image/tiff"> - <magic priority="50"> - <match value="0x4d4d2a00" type="string" offset="0" /> - <match value="0x49492a00" type="string" offset="0" /> - </magic> - </mime-type> - - <mime-type type="message/rfc822"> - <magic priority="50"> - <match type="string" value="Relay-Version:" offset="0" /> - <match type="string" value="#! rnews" offset="0" /> - <match type="string" value="N#! rnews" offset="0" /> - <match type="string" value="Forward to" offset="0" /> - <match type="string" value="Pipe to" offset="0" /> - <match type="string" value="Return-Path:" offset="0" /> - <match type="string" value="From:" offset="0" /> - <match type="string" value="Message-ID:" offset="0" /> - <match type="string" value="Date:" offset="0" /> - </magic> - </mime-type> - - <mime-type type="application/x-javascript"> - <glob pattern="*.js" /> - </mime-type> - - - <mime-type type="image/vnd.wap.wbmp"> - <_comment>Wireless Bitmap File Format</_comment> - <glob pattern="*.wbmp" /> - </mime-type> - - <mime-type type="image/x-psd"> - <alias type="image/photoshop" /> - </mime-type> - - <mime-type type="image/x-xcf"> - <alias type="image/xcf" /> - <magic priority="50"> - <match type="string" value="gimp xcf " offset="0" /> - </magic> - </mime-type> - - <mime-type type="application/x-shockwave-flash"> - <glob pattern="*.swf"/> - <magic priority="50"> - <match type="string" value="FWS" offset="0"/> - <match type="string" value="CWS" offset="0"/> - </magic> - </mime-type> - - <mime-type type="model/iges"> - <_comment> - Initial Graphics Exchange Specification Format - </_comment> - <glob pattern="*.igs" /> - <glob pattern="*.iges" /> - </mime-type> - - <mime-type type="model/mesh"> - <glob pattern="*.msh" /> - <glob pattern="*.mesh" /> - <glob pattern="*.silo" /> - </mime-type> - - <mime-type type="model/vrml"> - <glob pattern="*.vrml" /> - </mime-type> - - <mime-type type="text/x-tcl"> - <alias type="application/x-tcl" /> - </mime-type> - - <mime-type type="text/x-tex"> - <alias type="application/x-tex" /> - </mime-type> - - <mime-type type="text/x-texinfo"> - <alias type="application/x-texinfo" /> - </mime-type> - - <mime-type type="text/x-troff-me"> - <alias type="application/x-troff-me" /> - </mime-type> - - <mime-type type="video/vnd.mpegurl"> - <glob pattern="*.mxu" /> - </mime-type> - - <mime-type type="x-conference/x-cooltalk"> - <_comment>Cooltalk Audio</_comment> - <glob pattern="*.ice" /> - </mime-type> - -</mime-info> diff --git a/search-server/nutch/crawl_jsp/urls/urls.txt b/search-server/nutch/crawl_jsp/urls/urls.txt deleted file mode 100644 index 020ed3b..0000000 --- a/search-server/nutch/crawl_jsp/urls/urls.txt +++ /dev/null @@ -1,7 +0,0 @@ -file:///shared/repo/trunk/eng/docs/guides/channel-mgmt/en/ -file:///shared/repo/trunk/eng/docs/guides/client-config/en/ -file:///shared/repo/trunk/eng/docs/guides/proxy/rhn511/en/ -file:///shared/repo/trunk/eng/docs/guides/reference/en/ -file:///shared/repo/trunk/eng/docs/guides/satellite/en/ -file:///shared/repo/trunk/eng/docs/release-notes/satellite/rhn510/en-US/ - diff --git a/search-server/rhn_search_daemon_dev.conf b/search-server/rhn_search_daemon_dev.conf deleted file mode 100644 index 964fe46..0000000 --- a/search-server/rhn_search_daemon_dev.conf +++ /dev/null @@ -1,45 +0,0 @@ -wrapper.java.command=/usr/bin/java -wrapper.java.mainclass=com.redhat.satellite.search.Main -wrapper.java.classpath.1=%PWD%/build/ -wrapper.java.classpath.2=%PWD%/lib/commons-lang-2.0.jar -wrapper.java.classpath.3=%PWD%/lib/commons-logging-1.1.jar -wrapper.java.classpath.4=%PWD%/lib/ibatis-2.3.0.677.jar -wrapper.java.classpath.5=%PWD%/lib/log4j-1.2.8.jar -wrapper.java.classpath.6=%PWD%/lib/lucene-core-2.3.2.jar -wrapper.java.classpath.7=%PWD%/lib/ojdbc14-1.10.jar -wrapper.java.classpath.8=%PWD%/lib/picocontainer-1.3.jar -wrapper.java.classpath.9=%PWD%/lib/quartz-1.5.1.jar -wrapper.java.classpath.10=%PWD%/lib/redstone-xmlrpc-1.1_20071120.jar -wrapper.java.classpath.11=%PWD%/lib/redstone-xmlrpc-client-1.1_20071120.jar -wrapper.java.classpath.12=%PWD%/lib/simple-core-3.1.3.jar -wrapper.java.classpath.13=/usr/share/java/tanukiwrapper.jar -wrapper.java.classpath.14=%PWD%/lib/logdriver-0.1.jar -wrapper.java.classpath.15=%PWD%/lib/lucene-analyzers-2.3.2.jar -wrapper.java.classpath.16=/usr/share/nutch -wrapper.java.classpath.17=/usr/share/nutch/conf -wrapper.java.classpath.18=%PWD%/lib/nutch-2008-12-01_04-01-21.jar -wrapper.java.classpath.19=%PWD%/lib/hadoop-0.18.1-core.jar -wrapper.java.classpath.20=%PWD%/lib/commons-httpclient-3.0.jar -wrapper.java.classpath.21=%PWD%/lib/oro-2.0.8.jar -wrapper.java.classpath.22=%PWD%/lib/icu4j-3_8_1.jar -wrapper.java.classpath.23=%PWD%/lib/icu4j-charsets-3_8_1.jar -wrapper.java.classpath.24=%PWD%/lib/commons-cli-1.0.jar -wrapper.java.classpath.25=%PWD%/lib/commons-codec-1.2.jar -wrapper.java.classpath.26=%PWD%/lib/tika-0.1-incubating.jar - - -wrapper.java.library.path.1=/usr/lib -wrapper.java.library.path.2=/usr/lib64 -wrapper.console.format=PM -wrapper.console.loglevel=INFO -wrapper.logfile=%PWD%/rhn_search_daemon.log -wrapper.logfile.format=LPTM -wrapper.logfile.loglevel=DEBUG -wrapper.logfile.maxsize=5m -wrapper.logfile.maxfiles=0 -wrapper.syslog.loglevel=DEBUG - -wrapper.java.additional.1=-Dsearch.config.dir=%PWD%/src/config/search -wrapper.java.additional.2=-Xdebug -wrapper.java.additional.3=-Xnoagent -wrapper.java.additional.4=-Xrunjdwp:transport=dt_socket,address=9999,server=y,suspend=n diff --git a/search-server/run.sh b/search-server/run.sh deleted file mode 100755 index 0bbf58a..0000000 --- a/search-server/run.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -rpm -q tanukiwrapper > /dev/null -RETURN_VAL=$? -if [ ! "$RETURN_VAL" -eq "0" ]; then - echo "Please install tanukiwrapper: up2date tanukiwrapper or yum install tanukiwrapper" - exit 1; -fi - -if [ ! -r "/etc/rhn/search/rhn_search.conf" ]; then - echo "you need to symlink rhn_search.conf as /etc/rhn/search/rhn_search.conf" - exit 2; -fi - -echo "starting server" -export CLASSPATH=`pwd`/build -#java -Djava.library.path=/usr/lib -classpath `build-classpath-directory lib dist` com.redhat.satellite.search.Main -/usr/sbin/tanukiwrapper `pwd`/rhn_search_daemon_dev.conf wrapper.pidfile=`pwd`/rhn-search.pid wrapper.daemonize=FALSE diff --git a/search-server/scripts/Reindex.groovy b/search-server/scripts/Reindex.groovy deleted file mode 100644 index 49e94e0..0000000 --- a/search-server/scripts/Reindex.groovy +++ /dev/null @@ -1,43 +0,0 @@ -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.WriteQuery; -import com.redhat.satellite.search.config.Configuration; - -def deleteQuery(config, queryName) { - DatabaseManager databaseManager = new DatabaseManager(config) - WriteQuery query = null; - try { - query = databaseManager.getWriterQuery(queryName); - query.delete(null); - } - finally { - query.close(); - } -} - -/** - Currently, only handles Errata and Packages - Need to add: User, System, Docs - **/ -Configuration config = new Configuration() -deleteQueries = ["deleteLastErrata", "deleteLastPackage", - "deleteLastServer", "deleteLastHardwareDevice", - "deleteLastSnapshotTag", "deleteLastServerCustomInfo"] -deleteQueries.each{deleteQuery(config, it)} - -println "Database has been prepared so we can re-index." - -indexWorkDir = config.getString("search.index_work_dir", null); -println "Previous indexes will be deleted on filesystem under ${indexWorkDir}" - -dirs = [indexWorkDir+"/package", indexWorkDir+"/errata", - indexWorkDir+"/server", indexWorkDir+"/hwdevice", - indexWorkDir+"/snapshotTag", indexWorkDir+"/serverCustomInfo"] -dirs.each { - cmd = "rm -r $it" - def proc = cmd.execute() - proc.waitFor() -} - - - - diff --git a/search-server/scripts/build.properties b/search-server/scripts/build.properties deleted file mode 100644 index 6dcc9ac..0000000 --- a/search-server/scripts/build.properties +++ /dev/null @@ -1,22 +0,0 @@ -java.src.dir=src/java -config.src.dir=src/config -java.lib.dir=lib -java.task.lib.dir=../task-lib -report.dir=report -build.dir=build -build.instrument.dir=instrumented -dist.dir=dist -jar.name=spacewalk-search-scripts -jar.version=1.0 -jar.file.name=${jar.name}-${jar.version}.jar -ivyserver= http://jmrodri.fedorapeople.org -ivy.log.module.when.found=false -ivy.log.resolved.revision=false -# Checkstyle config entries -checkstyle.cache.file=.checkstyle_cache -checkstyle.header.file=./buildconf/LICENSE.txt -javadoc.method.scope=public -javadoc.type.scope=package -javadoc.var.scope=package -javadoc.lazy=true -jpackage.jars="commons-lang commons-logging redstone-xmlrpc redstone-xmlrpc-client quartz tanukiwrapper log4j commons-httpclient oro commons-cli commons-codec" diff --git a/search-server/scripts/build.xml b/search-server/scripts/build.xml deleted file mode 100644 index f45e670..0000000 --- a/search-server/scripts/build.xml +++ /dev/null @@ -1,256 +0,0 @@ -<project name="search-server" basedir="." default="all"> - <import file="./buildconf/build-utils.xml" /> - - <property file="build.properties" /> - - <path id="project.taskjars"> - <fileset dir="${java.task.lib.dir}" includes="**/*.jar" /> - </path> - - <path id="project.classpath"> - <fileset dir="${java.lib.dir}" includes="**/*.jar" /> - <fileset dir="${java.task.lib.dir}" > - <include name="junit*.jar" /> - <include name="emma*.jar" /> - </fileset> - </path> - - <path id="test.classpath"> - <pathelement location="${build.dir}" /> - <path refid="project.classpath" /> - <pathelement location="${basedir}/src/config" /> - </path> - - <path id="run.classpath"> - <path refid="project.classpath" /> - <pathelement location="${dist.dir}/${jar.file.name}"/> - </path> - - <target name="init-tasks"> - <taskdef name="ivy-retrieve" - classname="fr.jayasoft.ivy.ant.IvyRetrieve" - classpathref="project.taskjars"/> - <taskdef name="ivy-configure" - classname="fr.jayasoft.ivy.ant.IvyConfigure" - classpathref="project.taskjars"/> - <taskdef name="ivy-resolve" - classname="fr.jayasoft.ivy.ant.IvyResolve" - classpathref="project.taskjars"/> - <taskdef name="ivy-publish" - classname="fr.jayasoft.ivy.ant.IvyPublish" - classpathref="project.taskjars"/> - <taskdef resource="checkstyletask.properties"> - <classpath> - <pathelement path="${java.task.lib.dir}/checkstyle-4.1.jar" /> - <pathelement path="${java.task.lib.dir}/commons-beanutils-1.7.0.jar" /> - <pathelement path="${java.task.lib.dir}/commons-logging-1.0.4.jar" /> - <pathelement path="${java.task.lib.dir}/antlr-2.7.6.jar" /> - <pathelement path="${java.task.lib.dir}/regexp-1.3.jar" /> - </classpath> - </taskdef> - <taskdef resource="emma_ant.properties" classpathref="project.taskjars" /> - <taskdef resource="net/sf/antcontrib/antlib.xml"> - <classpath> - <pathelement location="${java.task.lib.dir}/ant-contrib-1.0.jar"/> - </classpath> - </taskdef> - </target> - - <target name="check-testcase-name" if="testcase" depends="init-tasks"> - <propertyregex property="valid.testcase" - input="${testcase}" - regexp="(.*)Test$" - select="\1" - casesensitive="true" /> - </target> - - <target name="check-testcase" depends="check-testcase-name" if="testcase" - unless="valid.testcase"> - <fail message="Invalid testcase name: ${testcase}. Class name must look like .*Test"/> - </target> - - <target name="test" depends="compile, check-testcase" - description="Run the tests (use -Dtestcase=<testcase>)"> - <mkdir dir="${report.dir}" /> - <!-- set testcase to wildcard "*" to test everything, unless - a specific testcase name was supplied. --> - <if> - <not> - <isset property="${testcase}" /> - </not> - <then> - <property name="testcase" value="*Test"/> - </then> - </if> - <junit> - <sysproperty key="rhn.config.dir" value="/etc/rhn" /> - <sysproperty key="search.config.dir" value="src/config/search" /> - <sysproperty key="log4j.configuration" value="log4j.test.properties" /> - <classpath refid="test.classpath" /> - <formatter type="brief" usefile="false" /> - <formatter type="xml" /> - <batchtest fork="yes" todir="${report.dir}"> - <fileset dir="${build.dir}" includes="**/${testcase}.class" /> - </batchtest> - </junit> - </target> - - <target name="checkstyle" depends="init-tasks, package"> - <delete dir="${report.dir}" /> - <mkdir dir="${report.dir}" /> - <checkstyle config="checkstyle.xml" failOnViolation="true" > - <property key="javadoc.method.scope" value="public" /> - <property key="javadoc.type.scope" value="package" /> - <property key="javadoc.var.scope" value="package" /> - <property key="javadoc.lazy" value="${javadoc.lazy}" /> - <fileset dir="${java.src.dir}" > - <include name="**/*.java" /> - <exclude name="**/tests/**" /> - </fileset> - <classpath> - <fileset dir="${java.lib.dir}" includes="**/*.jar" /> - <fileset dir="${java.task.lib.dir}" includes="junit*.jar" /> - <fileset dir="${dist.dir}" includes="${jar.file.name}" /> - </classpath> - <formatter type="plain" /> - <formatter type="xml" toFile="${report.dir}/checkstyle_report.xml" /> - </checkstyle> - </target> - - <target name="resolve-local" description="resolve jars via jpackage" if="installbuild"> - <copy toDir="${java.lib.dir}" overwrite="true"> - <fileset dir="buildconf/tempjars"> - <include name="**/*.jar" /> - </fileset> - </copy> - - <jpackage-deps jars="${jpackage.jars}" dir="${java.lib.dir}" /> - </target> - - <target name="resolve-ivy" description="retrieve dependencies with ivy" - depends="init-tasks" unless="installbuild"> - <!-- properties set here to show we are overriding default ivy values --> - <property name="ivy.default.ivy.user.dir" value="${user.home}/.ivy/search-server"/> - <property name="ivy.local.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> - <property name="ivy.ivyrep.default.ivy.root" value="${ivyserver}/ivy/" /> - <property name="ivy.ivyrep.default.artifact.root" value="${ivyserver}/ivy/" /> - <property name="ivy.ivyrep.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> - <ivy-retrieve/> - </target> - - <target name="resolve" depends="resolve-ivy,resolve-local" /> - - <target name="prep"> - <mkdir dir="${build.dir}" /> - <mkdir dir="${dist.dir}" /> - <mkdir dir="${java.lib.dir}" /> - </target> - - <target name="clean"> - <delete dir="${build.dir}" /> - <delete dir="${build.instrument.dir}" /> - <delete dir="${dist.dir}" /> - <delete dir="${java.lib.dir}" /> - <delete dir="${report.dir}" /> - <delete file="coverage.ec" /> - <delete file="rhn_search_daemon.log" /> - </target> - - <target name="compile" depends="prep, resolve"> - <javac srcdir="${java.src.dir}" - classpathref="project.classpath" - destdir="${build.dir}" - deprecation="on" - excludesfile="buildconf/exclude" - debug="on" /> - <copy file="${config.src.dir}/log4j.properties" todir="${build.dir}" /> - </target> - - <target name="package" depends="compile"> - <jar basedir="${build.dir}" - destfile="${dist.dir}/${jar.file.name}" - excludes="**/tests/**" - includes="**/**" /> - </target> - - <target name="create-local-repo" description="Create a local Ivy repo"> - <mkdir dir="${user.home}/.ivy/search-server/local/" /> - </target> - - <target name="clean-cache" description="Removes the Ivy jar cache"> - <delete dir="${lib.dir}" quiet="true" /> - <delete dir="${user.home}/.ivy/search-server/cache/" quiet="true" /> - </target> - - <target name="all" depends="package" /> - - <target name="init-install"> - <property name="installbuild" value="true"/> - </target> - - <target name="install" depends="init-install,package" /> - - <target name="instrument" depends="init-tasks"> - <mkdir dir="${report.dir}" /> - <mkdir dir="${build.instrument.dir}" /> - - <emma enabled="true"> - <instr merge="no" - mode="overwrite" - instrpath="${build.dir}" - outfile="${report.dir}/coverage.em" > - <filter excludes="*Exception" /> - <filter excludes="*.tests.*" /> - </instr> - <!-- - <instr outdir="${build.instrument.dir}" merge="no" - instrpath="${build.dir}" - mode="fullcopy" - outfile="${report.dir}/coverage.em" > - <filter excludes="*Exception" /> - <filter excludes="*.tests.*" /> - </instr> - --> - </emma> - - <!-- - <copy toDir="${build.instrument.dir}/classes"> - <fileset dir="${build.dir}"> - <include name="**/*.xml" /> - </fileset> - </copy> - --> - </target> - - <target name="coverage-report" depends="instrument"> - <emma enabled="true"> - <report sourcepath="${java.src.dir}" - sort="+block,+name,+method,+class" - metrics="method:70,block:80,line:80,class:100"> - <fileset dir="${report.dir}" includes="*.em" /> - <fileset dir="." includes="*.ec" /> - <html outfile="${report.dir}/emma.html" - depth="method" - columns="name,class,method,block,line" /> - <xml outfile="${report.dir}/emma.xml" - depth="method" - columns="name,class,method,block,line" /> - </report> - </emma> - </target> - - - <target name="indexDocs" depends="package" - description="Crawls help docs with nutch and generates a lucene index"> - <java - classname="com.redhat.satellite.search.index.docs.WebCrawl" - dir="${basedir}" - fork="true" - failonerror="true"> - - <arg value="-h"/> - <classpath refid="run.classpath" /> - </java> - </target> - -</project> diff --git a/search-server/scripts/buildconf/LICENSE.txt b/search-server/scripts/buildconf/LICENSE.txt deleted file mode 100644 index 629462a..0000000 --- a/search-server/scripts/buildconf/LICENSE.txt +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ diff --git a/search-server/scripts/buildconf/build-utils.xml b/search-server/scripts/buildconf/build-utils.xml deleted file mode 100644 index d428618..0000000 --- a/search-server/scripts/buildconf/build-utils.xml +++ /dev/null @@ -1,19 +0,0 @@ -<project name="build-utils.xml"> - <!-- - creates symlinks for jpackage jars - requires: jpackage-utils - other: requires rpms of the jars specified to be installed - --> - <macrodef name="jpackage-deps"> - <attribute name="jars" /> - <attribute name="dir" /> - - <sequential> - <exec executable="build-jar-repository" failonerror="true" > - <arg value="--preserve-naming" /> - <arg value="-s @{dir}" /> - <arg line="@{jars}" /> - </exec> - </sequential> - </macrodef> -</project> diff --git a/search-server/scripts/buildconf/exclude b/search-server/scripts/buildconf/exclude deleted file mode 100644 index e69de29..0000000 diff --git a/search-server/scripts/checkstyle.xml b/search-server/scripts/checkstyle.xml deleted file mode 100644 index 2cb51d5..0000000 --- a/search-server/scripts/checkstyle.xml +++ /dev/null @@ -1,233 +0,0 @@ -<?xml version="1.0"?> -<!DOCTYPE module PUBLIC - "-//Puppy Crawl//DTD Check Configuration 1.1//EN" - "http://www.puppycrawl.com/dtds/configuration_1_1.dtd"> - -<!-- - - Checkstyle configuration that checks the sun coding conventions from: - - - the Java Language Specification at - http://java.sun.com/docs/books/jls/second_edition/html/index.html - - - the Sun Code Conventions at http://java.sun.com/docs/codeconv/ - - - the Javadoc guidelines at - http://java.sun.com/j2se/javadoc/writingdoccomments/index.html - - - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html - - - some best practices - - Checkstyle is very configurable. Be sure to read the documentation at - http://checkstyle.sf.net (or in your downloaded distribution). - - Most Checks are configurable, be sure to consult the documentation. - - To completely disable a check, just comment it out or delete it from the file. - - Finally, it is worth reading the documentation. - ---> - -<module name="Checker"> - - <!-- Checks that a package.html file exists for each package. --> - <!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml --> -<!-- <module name="PackageHtml"/> --> - - <!-- Checks whether files end with a new line. --> - <!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile --> - <module name="NewlineAtEndOfFile"/> - - <!-- Checks that property files contain the same keys. --> - <!-- See http://checkstyle.sf.net/config_misc.html#Translation --> - <module name="Translation"/> - - <module name="TreeWalker"> - - <property name="cacheFile" value="${checkstyle.cache.file}"/> - - <!-- ************************************************************** --> - <!-- Checks that are different from the sun coding conventions ones --> - <!-- ************************************************************** --> - <property name="tabWidth" value="4"/> - <module name="LeftCurly" /> - <module name="RightCurly"> - <property name="option" value="alone"/> - </module> - <module name="LineLength"> - <property name="ignorePattern" value="import"/> - <property name="max" value="92"/> - </module> - - <!-- ************************************************************** --> - <!-- Default Sun coding conventions checks --> - <!-- ************************************************************** --> - - <!-- Checks for Javadoc comments. --> - <!-- See http://checkstyle.sf.net/config_javadoc.html --> - <module name="JavadocMethod"> - <property name="scope" value="${javadoc.method.scope}" /> - <property name="allowUndeclaredRTE" value="true" /> - <property name="allowThrowsTagsForSubclasses" value="${javadoc.lazy}" /> - <property name="allowMissingParamTags" value="${javadoc.lazy}" /> - <property name="allowMissingThrowsTags" value="${javadoc.lazy}" /> - <property name="allowMissingReturnTag" value="${javadoc.lazy}" /> - </module> - <module name="JavadocType"> - <property name="versionFormat" value="\$Rev.*\$"/> - <property name="scope" value="${javadoc.type.scope}" /> - </module> - <module name="JavadocVariable"> - <property name="scope" value="${javadoc.var.scope}" /> - <property name="severity" value="ignore"/> - </module> - -<!-- JavadocStyle not supported by some versions. --> -<!-- <module name="JavadocStyle"/> --> - - - <!-- Checks for Naming Conventions. --> - <!-- See http://checkstyle.sf.net/config_naming.html --> - <module name="ConstantName"/> - <module name="LocalFinalVariableName"/> - <module name="LocalVariableName"/> - <module name="MethodName"/> - <module name="PackageName"> - <property name="format" value="^[a-z]+(\.[A-Za-z][A-Za-z0-9]*)*$"/> - </module> - <module name="ParameterName"/> - <module name="StaticVariableName"/> - <module name="TypeName"/> - - - <!-- Checks for Headers --> - <!-- See http://checkstyle.sf.net/config_header.html --> - <module name="Header"> - <!-- The follow property value demonstrates the ability --> - <!-- to have access to ANT properties. In this case it uses --> - <!-- the ${basedir} property to allow Checkstyle to be run --> - <!-- from any directory within a project. --> - <property name="headerFile" value="${checkstyle.header.file}"/> - </module> - - <!-- Following interprets the header file as regular expressions. --> - <!-- <module name="RegexpHeader"/> --> - - - - <!-- Checks for imports --> - <!-- See http://checkstyle.sf.net/config_import.html --> - <module name="AvoidStarImport"/> - <module name="IllegalImport"/> <!-- defaults to sun.* packages --> - <module name="RedundantImport"/> - <module name="UnusedImports"/> - - - <!-- Checks for Size Violations. --> - <!-- See http://checkstyle.sf.net/config_sizes.html --> - <module name="MethodLength"> - <property name="tokens" value="METHOD_DEF"/> - <property name="max" value="150"/> - <property name="countEmpty" value="false"/> - </module> - <module name="ParameterNumber"/> - - - <!-- Checks for whitespace --> - <!-- See http://checkstyle.sf.net/config_whitespace.html --> - <module name="EmptyForIteratorPad"/> - <module name="NoWhitespaceAfter"> - <property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, UNARY_PLUS, TYPECAST, GENERIC_START"/> - </module> - <module name="NoWhitespaceBefore"> - <property name="tokens" value="SEMI, POST_DEC, POST_INC, GENERIC_START, GENERIC_END"/> - </module> - <module name="OperatorWrap"> - <property name="option" value="eol" /> - </module> - <module name="ParenPad"/> -<!-- <module name="TypecastParenPad"/> --> - <module name="TabCharacter"/> - <module name="WhitespaceAfter"> - <property name="tokens" value="COMMA, SEMI, GENERIC_END"/> - </module> - <module name="WhitespaceAround"> - <property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, -COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LCURLY, LE, LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, -LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS, -MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION, RCURLY, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN, STAR, STAR_ASSIGN, TYPE_EXTENSION_AND, WILDCARD_TYPE"/> - </module> - <module name="MethodParamPad"> - <property name="allowLineBreaks" value="true"/> - </module> - - - <!-- Modifier Checks --> - <!-- See http://checkstyle.sf.net/config_modifiers.html --> - <module name="ModifierOrder"/> - <module name="RedundantModifier"/> - - - <!-- Checks for blocks. You know, those {}'s --> - <!-- See http://checkstyle.sf.net/config_blocks.html --> - <module name="AvoidNestedBlocks"/> - <module name="EmptyBlock"> - <property name="option" value="text"/> - </module> - - <module name="NeedBraces"/> - - - <!-- Checks for common coding problems --> - <!-- See http://checkstyle.sf.net/config_coding.html --> - <!-- <module name="AvoidInlineConditionals"/> --> -<!-- <module name="CovariantEquals"/> --> - <module name="DoubleCheckedLocking"/> <!-- MY FAVOURITE --> - <module name="EmptyStatement"/> - <module name="EqualsHashCode"/> - <module name="HiddenField"/> - <module name="IllegalInstantiation"/> - <module name="InnerAssignment"/> - <!-- <module name="MagicNumber" /> --> - <module name="MissingSwitchDefault"/> -<!-- <module name="RedundantThrows"/> --> - <module name="SimplifyBooleanExpression"/> - <module name="SimplifyBooleanReturn"/> -<!-- <module name="IllegalCatch"/> --> -<!-- <module name="PackageDeclaration"/> --> - - <!-- Checks for class design --> - <!-- See http://checkstyle.sf.net/config_design.html --> -<!-- <module name="DesignForExtension"/>--> -<!-- <module name="FinalClass"/> --> - <module name="HideUtilityClassConstructor"/> - <module name="InterfaceIsType"/> - <module name="VisibilityModifier"> - <property name="protectedAllowed" value="true" /> - </module> - - - <!-- Miscellaneous other checks. --> - <!-- See http://checkstyle.sf.net/config_misc.html --> - <module name="ArrayTypeStyle"/> -<!-- <module name="FinalParameters"/> --> -<!-- - <module name="GenericIllegalRegexp"> - <property name="format" value="System\.out\.println"/> - </module> ---> - <module name="UpperEll"/> - - <!-- Optional Checks --> - <!-- (gack, not available until version 3 of checkstyle --> -<!-- - <module name="usage.OneMethodPrivateField"/> - <module name="usage.UnusedLocalVariable"/> - <module name="usage.UnusedParameter"/> - <module name="usage.UnusedPrivateField"/> ---> - </module> - -</module> diff --git a/search-server/scripts/ivy.xml b/search-server/scripts/ivy.xml deleted file mode 100644 index cffd1c1..0000000 --- a/search-server/scripts/ivy.xml +++ /dev/null @@ -1,28 +0,0 @@ -<ivy-module version="1.0"> - <info organisation="redhat" module="rhn-java" /> - <dependencies> - <dependency org="redhat" name="simple-core" rev="3.1.3" /> - <dependency org="redhat" name="lucene-core" rev="2.3.2" /> - <dependency org="redhat" name="lucene-analyzers" rev="2.3.2" /> - <dependency org="redhat" name="commons-lang" rev="2.0" /> - <dependency org="redhat" name="commons-logging" rev="1.1" /> - <dependency org="redhat" name="redstone-xmlrpc" rev="1.1_20071120" /> - <dependency org="redhat" name="redstone-xmlrpc-client" rev="1.1_20071120" /> - <dependency org="redhat" name="ibatis" rev="2.3.0.677" /> - <dependency org="redhat" name="ojdbc14" rev="1.10" /> - <dependency org="redhat" name="quartz" rev="1.5.1" /> - <dependency org="redhat" name="picocontainer" rev="1.3" /> - <dependency org="redhat" name="tanukiwrapper" rev="3.1.2" /> - <dependency org="redhat" name="log4j" rev="1.2.8" /> - <dependency org="redhat" name="logdriver" rev="0.1" /> - <dependency org="redhat" name="nutch" rev="1.0dev_2008_05_14" /> - <dependency org="redhat" name="hadoop" rev="0.16.4-core" /> - <dependency org="redhat" name="commons-cli" rev="1.0" /> - <!-- dependency org="redhat" name="commons-httpclient" rev="3.0" / --> - <!-- dependency org="redhat" name="oro" rev="2.0.8" / --> - <!-- dependency org="redhat" name="commons-codec" rev="1.2" / --> - <!-- dependency org="redhat" name="tika" rev="0.1-incubating" / --> - <!-- dependency org="redhat" name="icu4j" rev="3_8_1" / --> - <!-- dependency org="redhat" name="icu4j-charsets" rev="3_8_1" / --> - </dependencies> -</ivy-module> diff --git a/search-server/scripts/lukeall-0.8.1.jar b/search-server/scripts/lukeall-0.8.1.jar deleted file mode 100644 index 347aee6..0000000 Binary files a/search-server/scripts/lukeall-0.8.1.jar and /dev/null differ diff --git a/search-server/scripts/maketar.sh b/search-server/scripts/maketar.sh deleted file mode 100755 index 6a389be..0000000 --- a/search-server/scripts/maketar.sh +++ /dev/null @@ -1,30 +0,0 @@ -if [ -a version ]; then - version=`cat version | awk '/^[0-9].[0-9].[0-9]* / { print $1 }'` - - if [ "$version" == "" ]; then - echo "Version not found" - exit -1 - fi - - echo "Creating /tmp/rhn-search-$version" - mkdir /tmp/rhn-search-$version - - echo "Copying repo to /tmp/rhn-search-$version/" - cp -R * /tmp/rhn-search-$version/ - - cd /tmp/ - rm -f rhn-search-$version/scripts/maketar.sh - echo "Creating /tmp/rhn-search-$version.tar.gz file" - tar -czf /tmp/rhn-search-$version.tar.gz rhn-search-$version/ --exclude .svn - cd - - - echo "Removing /tmp/rhn-search-$version" - rm -rf /tmp/rhn-search-$version - - echo "" - echo "You can find your tar at /tmp/rhn-search-$version.tar.gz" - echo "" - -else - echo "You moron, you gotta run this from the directory containing the version file." -fi diff --git a/search-server/scripts/reindex.sh b/search-server/scripts/reindex.sh deleted file mode 100755 index cc7b312..0000000 --- a/search-server/scripts/reindex.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env sh - -RHN_SEARCH_HOME="../" - -for f in $RHN_SEARCH_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done -for f in $RHN_SEARCH_HOME/dist/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done -export CLASSPATH - -groovy Reindex.groovy - - diff --git a/search-server/scripts/search.admin.updateIndex.test.py b/search-server/scripts/search.admin.updateIndex.test.py deleted file mode 100755 index 8962c9d..0000000 --- a/search-server/scripts/search.admin.updateIndex.test.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python - -import sys -import xmlrpclib - -if __name__ == "__main__": - if len(sys.argv) < 2: - print "Usage: %s INDEX_NAME" % (sys.argv[0]) - sys.exit(1) - port = 2828 - addr = "http://127.0.0.1:%s" % (port) - client = xmlrpclib.ServerProxy(addr) - indexName = sys.argv[1] - result = client.admin.updateIndex(indexName) - print "Return from admin.updateIndex(%s) = %s" % (indexName, result) diff --git a/search-server/scripts/search.py b/search-server/scripts/search.py deleted file mode 100755 index c8d0a04..0000000 --- a/search-server/scripts/search.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python - -import xmlrpclib -from optparse import OptionParser - -indexName = "package" -usage = "usage: %prog [options] search term" -desc = "%prog searches for package (default) or systems with the given \ -search criteria" - -parser = OptionParser(usage=usage, description=desc) -parser.add_option("--sessionid", dest="sessionid", type="int", help="PXT sessionid") -parser.add_option("--package", action="store_true", dest="package", - help="search packages", default=True) -parser.add_option("--system", action="store_true", dest="system", - help="search systems", default=False) -parser.add_option("--indexName", dest="indexName", type="string", - help="lucene index name to search ex: package server hwdevice snapshotTag errata") -parser.add_option("--serverAddr", dest="serverAddr", type="string", default="localhost", - help="Server to authenticate to, NOT WHERE SEARCH SERVER RUNS") -parser.add_option("--username", dest="username", type="string", help="username") -parser.add_option("--password", dest="password", type="string", help="password") -parser.add_option("--debug", action="store_true", dest="debug", default=False, - help="enable debug output") - -(options, terms) = parser.parse_args() -if len(terms) < 1: - parser.error("please supply a search term\n" + str(parser.print_help())) - -if not options.sessionid and (not options.username or not options.password): - print parser.print_help() - parser.exit() - - -if options.package: - indexName = "package" - -if options.system: - indexName = "server" - -if options.indexName: - indexName = options.indexName - - -sessionid = None -if options.sessionid: - sessionid = options.sessionid - print "Using passed in authentication info, sessionid = %s" % (sessionid) -else: - xmlrpcURL = "http://%s/rhn/rpc/api" % (options.serverAddr) - print "Getting authentication information from: %s" % (xmlrpcURL) - rhnclient = xmlrpclib.Server(xmlrpcURL) - authSessionId = rhnclient.auth.login(options.username, options.password) - sessionid = int(authSessionId.split('x')[0]) - -url = "http://localhost:2828/RPC2" -print "Connecting to SearchServer: (%s)" % url -client = xmlrpclib.Server(url, verbose=options.debug) - -term = " ".join(terms) -print "searching for (%s) matching criteria: (%s)" % (indexName, str(term)) -items = client.index.search(sessionid, indexName, term) - -print "We got (%d) items back." % len(items) -print items - -#Remember to logout if the user didn't supply the sessionid -if not options.sessionid: - rhnclient.auth.logout(authSessionId) diff --git a/search-server/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java b/search-server/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java deleted file mode 100644 index 7a9f671..0000000 --- a/search-server/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java +++ /dev/null @@ -1,304 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.docs; - -import org.apache.nutch.crawl.Injector; -import org.apache.nutch.crawl.Generator; -import org.apache.nutch.crawl.CrawlDb; -import org.apache.nutch.crawl.LinkDb; -import org.apache.nutch.fetcher.Fetcher; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; -import org.apache.nutch.parse.ParseSegment; -import org.apache.nutch.indexer.DeleteDuplicates; -import org.apache.nutch.indexer.IndexMerger; -import org.apache.nutch.indexer.Indexer; -import org.apache.nutch.util.HadoopFSUtil; -import org.apache.nutch.util.NutchConfiguration; -import org.apache.nutch.util.NutchJob; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.PosixParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; - -import org.apache.commons.cli.ParseException; - -import org.apache.log4j.Logger; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.io.IOException; - - -/** -Crawls a set of urls and indexes their content. -Based off of nutch's crawl - * - * @version $Rev: $ - */ -public class WebCrawl { - private static Logger log = Logger.getLogger(WebCrawl.class); - - private Configuration conf; - private JobConf job; - private String inputUrlFile; - private String tmpCrawlDir; - private String outputIndexDir; - - private int depth; - private int threads; - private long topN; - - /** - * Constructor - */ - public WebCrawl() { - conf = NutchConfiguration.create(); - conf.addResource("crawl-tool.xml"); - job = new NutchJob(conf); - threads = job.getInt("fetcher.threads.fetch", 10); - depth = 5; - topN = Long.MAX_VALUE; - } - - /** - * @return Returns the dir used to store tmp crawl data - */ - public String getTmpCrawlDir() { - return tmpCrawlDir; - } - /** - * @param tmpCrawlDirIn the dir used to store tmp crawl data - */ - public void setTmpCrawlDir(String tmpCrawlDirIn) { - tmpCrawlDir = tmpCrawlDirIn; - } - /** - * @return Returns the file used to seed the url crawl - */ - public String getInputUrlFile() { - return inputUrlFile; - } - /** - * @param inputUrlFileIn file used to seed the url crawl - */ - public void setInputUrlFile(String inputUrlFileIn) { - inputUrlFile = inputUrlFileIn; - } - /** - * @return Returns the dir to store the index of the crawled docs - */ - public String getOutputIndexDir() { - return outputIndexDir; - } - /** - * @param outputIndexDirIn the dir to store the index of the crawled docs - */ - public void setOutputIndexDir(String outputIndexDirIn) { - outputIndexDir = outputIndexDirIn; - } - /** - * @return Returns the maximum depth of recursion allowed - */ - public int getDepth() { - return depth; - } - /** - * @param depthIn the maximum depth of recursion allowed - */ - public void setDepth(int depthIn) { - depth = depthIn; - } - /** - * @return Returns the maximum number of fetcher threads used - */ - public int getThreads() { - return threads; - } - /** - * @param threadsIn the maximum number of fetcher threads used - */ - public void setThreads(int threadsIn) { - threads = threadsIn; - } - /** - * @return Returns the maximum number of "out links" to follow in any given page - */ - public long getTopN() { - return topN; - } - /** - * @param topNIn the maximum number of "out links" to follow in any given page - */ - public void setTopN(long topNIn) { - topN = topNIn; - } - - protected String getDate() { - return new SimpleDateFormat("yyyyMMddHHmmss").format( - new Date(System.currentTimeMillis())); - } - - /** - * Uses nutch to crawl a list of ulrs defined in "inputUrlFile" - * Temporary scratch storage is held at "tmpCrawlDir" - * The desired output is an index of crawled pages, stored at "outputIndexDir" - * - * @return true when urls successfully crawled/indexed - */ - @SuppressWarnings("deprecation") - public boolean crawl() throws IOException { - log.info("Performing crawl with following config options: "); - log.info("inputUrlFile = " + inputUrlFile); - log.info("tmpCrawlDir = " + tmpCrawlDir); - log.info("outputIndexDir = " + outputIndexDir); - log.info("threads = " + threads); - log.info("depth = " + depth); - log.info("topN = " + topN); - - Path inputPath = new Path(inputUrlFile); - Path linkDb = new Path(tmpCrawlDir + "/linkdb"); - Path segments = new Path(tmpCrawlDir + "/segments"); - Path indexes = new Path(tmpCrawlDir + "/indexes"); - Path index = new Path(outputIndexDir); - Path crawlDb = new Path(tmpCrawlDir + "/crawldb"); - - Path tmpDir = job.getLocalPath("crawl" + Path.SEPARATOR + getDate()); - Injector injector = new Injector(conf); - Generator generator = new Generator(conf); - Fetcher fetcher = new Fetcher(conf); - ParseSegment parseSegment = new ParseSegment(conf); - CrawlDb crawlDbTool = new CrawlDb(conf); - LinkDb linkDbTool = new LinkDb(conf); - Indexer indexer = new Indexer(conf); - DeleteDuplicates dedup = new DeleteDuplicates(conf); - IndexMerger merger = new IndexMerger(conf); - - FileSystem fs = FileSystem.get(job); - - log.info("Create a new database (crawlDB) of link information"); - injector.inject(crawlDb, inputPath); - - for (int i = 0; i < depth; i++) { - log.info("Generate a fetch list from info in the crawlDB."); - Path segment = generator.generate(crawlDb, segments, -1, topN, - System.currentTimeMillis()); - if (segment == null) { - log.info("Stopping at depth = " + i + " instead of " + (depth - 1) + - " - no more URLs to fetch."); - break; - } - log.info("Fetch links"); - fetcher.fetch(segment, threads); // fetch it - if (!Fetcher.isParsing(job)) { - log.info("Parsing Segment"); - parseSegment.parse(segment); // parse it, if needed - } - Path[] p = new Path[1]; - p[0] = segment; - log.info("Update CrawlDB"); - crawlDbTool.update(crawlDb, p, true, true); // update crawldb - } - - linkDbTool.invert(linkDb, segments, true, true, false); // invert links - - // Delete old indexes - if (fs.exists(indexes)) { - log.info("Deleting old indexes: $indexes"); - fs.delete(indexes); - } - - // Delete old index - if (fs.exists(index)) { - log.info("Deleting old merged index: $index"); - fs.delete(index); - } - - // index, dedup & merge - indexer.index(indexes, crawlDb, linkDb, - fs.listPaths(segments, HadoopFSUtil.getPassAllFilter())); - - Path[] p = new Path[1]; - p[0] = indexes; - dedup.dedup(p); - merger.merge(fs.listPaths(indexes, HadoopFSUtil.getPassAllFilter()), - index, tmpDir); - log.info("Crawl finished"); - return true; - } - - - /** - * Performs a web crawl: - * @param args command line arguments - */ - public static void main(String[] args) throws IOException { - - WebCrawl wCrawl = new WebCrawl(); - - Options options = new Options(); - options.addOption("i", "inputUrlFile", true, - "file holding 'urls' file to seed web page crawling"); - options.addOption("o", "outputDir", true, "temp crawl output dir"); - options.addOption("x", "docsIndexDir", true, "docs index output dir"); - options.addOption("t", "threads", true, "number of threads"); - options.addOption("d", "depth", true, "depth to recurse towards"); - options.addOption("n", "topN", true, "maximum number of out links to follow"); - options.addOption("h", "help", false, "print help message"); - - CommandLineParser parser = new PosixParser(); - try { - CommandLine line = parser.parse(options, args); - - if (line.hasOption("h")) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("WebCrawl", options); - return; - } - if (line.hasOption("i")) { - wCrawl.setInputUrlFile(line.getOptionValue("i")); - } - if (line.hasOption("x")) { - wCrawl.setOutputIndexDir(line.getOptionValue("x")); - } - if (line.hasOption("o")) { - wCrawl.setTmpCrawlDir(line.getOptionValue("o")); - } - if (line.hasOption("t")) { - wCrawl.setThreads(Integer.parseInt(line.getOptionValue("t"))); - } - if (line.hasOption("d)")) { - wCrawl.setDepth(Integer.parseInt(line.getOptionValue("d"))); - } - if (line.hasOption("n")) { - wCrawl.setTopN(Integer.parseInt(line.getOptionValue("n"))); - } - - } - catch (ParseException exp) { - System.err.println("Parsing failed. Reason: " + exp.getMessage()); - } - - if (!wCrawl.crawl()) { - System.err.println("Error -- WebCrawl.crawl() Failed!"); - } - } - - -} diff --git a/search-server/scripts/webCrawl.sh b/search-server/scripts/webCrawl.sh deleted file mode 100755 index da352f6..0000000 --- a/search-server/scripts/webCrawl.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -SEARCH_HOME=.. -NUTCH_HOME=/usr/share/nutch - -CLASSPATH=${CLASSPATH}:${SEARCH_HOME}/build -#Put our nutch/conf dir before the NUTCH_HOME so ensure our config files are used -CLASSPATH=${CLASSPATH}:${SEARCH_HOME}/nutch/conf -CLASSPATH=${CLASSPATH}:${NUTCH_HOME} -for f in ${SEARCH_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -java -cp ${CLASSPATH} com.redhat.satellite.search.scheduler.tasks.crawl.WebCrawl --inputUrlFile ${SEARCH_HOME}/nutch/urls --outputDir ./crawl_output --docsIndexDir ./docsIndexDir --depth 2 --threads 4 - - - - diff --git a/search-server/spacewalk-search.spec b/search-server/spacewalk-search.spec deleted file mode 100644 index fcc3580..0000000 --- a/search-server/spacewalk-search.spec +++ /dev/null @@ -1,255 +0,0 @@ -%{!?__redhat_release:%define __redhat_release UNKNOWN} - -Name: spacewalk-search -Summary: Spacewalk Full Text Search Server -Group: Applications/Internet -License: GPLv2 -Version: 0.8.1 -Release: 1%{?dist} -# This src.rpm is cannonical upstream -# You can obtain it using this set of commands -# git clone git://git.fedorahosted.org/git/spacewalk.git/ -# cd search-server -# make test-srpm -URL: https://fedorahosted.org/spacewalk -Source0: %{name}-%{version}.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildArch: noarch - -#Requires: apache-ibatis-sqlmap -Requires: doc-indexes -Requires: jakarta-commons-cli -Requires: jakarta-commons-codec -Requires: jakarta-commons-httpclient -Requires: jakarta-commons-lang >= 0:2.1 -Requires: jakarta-commons-logging -Requires: jpackage-utils >= 0:1.5 -Requires: log4j -Requires: oro -#Requires: lucene -Requires: quartz -Requires: redstone-xmlrpc -#Requires: picocontainer -Requires: tanukiwrapper -Obsoletes: rhn-search < 5.3.0 -BuildRequires: ant -#BuildRequires: apache-ibatis-sqlmap -BuildRequires: jakarta-commons-cli -BuildRequires: jakarta-commons-codec -BuildRequires: jakarta-commons-httpclient -BuildRequires: jakarta-commons-lang >= 0:2.1 -BuildRequires: jakarta-commons-logging -BuildRequires: java-devel >= 1.6.0 -BuildRequires: log4j -BuildRequires: oro -#BuildRequires: lucene -BuildRequires: quartz -BuildRequires: redstone-xmlrpc -#BuildRequires: picocontainer -BuildRequires: tanukiwrapper -Requires(post): chkconfig -Requires(preun): chkconfig -# This is for /sbin/service -Requires(preun): initscripts - -%description -This package contains the code for the Full Text Search Server for -Spacewalk Server. - -%prep -%setup -n %{name}-%{version} - -%install -rm -fr ${RPM_BUILD_ROOT} -ant -Djar.version=%{version} install -install -d -m 755 $RPM_BUILD_ROOT%{_sysconfdir}/rhn/search -install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search -install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/indexes -install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/lib -install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/classes/com/redhat/satellite/search/db -install -d -m 755 $RPM_BUILD_ROOT%{_initrddir} -install -d -m 755 $RPM_BUILD_ROOT%{_bindir} -install -d -m 755 $RPM_BUILD_ROOT%{_var}/log/rhn/search -install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/nutch -install -d -m 755 $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d -install -p -m 644 dist/%{name}-%{version}.jar $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/lib/ -# using install -m does not preserve the symlinks -cp -d lib/* $RPM_BUILD_ROOT/%{_prefix}/share/rhn/search/lib -install -p -m 644 src/config/log4j.properties $RPM_BUILD_ROOT/%{_prefix}/share/rhn/search/classes/log4j.properties -install -p -m 644 src/config/etc/logrotate.d/rhn-search $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d/rhn-search -install -p -m 644 src/config/com/redhat/satellite/search/db/* $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/classes/com/redhat/satellite/search/db -install -p -m 755 src/config/rhn-search $RPM_BUILD_ROOT%{_initrddir} -ln -s -f /usr/sbin/tanukiwrapper $RPM_BUILD_ROOT%{_bindir}/rhnsearchd -install -p -m 644 src/config/search/rhn_search.conf $RPM_BUILD_ROOT%{_sysconfdir}/rhn/search/rhn_search.conf -install -p -m 644 src/config/search/rhn_search_daemon.conf $RPM_BUILD_ROOT%{_sysconfdir}/rhn/search/rhn_search_daemon.conf -ln -s -f %{_prefix}/share/rhn/search/lib/spacewalk-search-%{version}.jar $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/lib/spacewalk-search.jar - -%clean -rm -rf $RPM_BUILD_ROOT - -%post -# This adds the proper /etc/rc*.d links for the script -/sbin/chkconfig --add rhn-search - -%preun -if [ $1 = 0 ] ; then - /sbin/service rhn-search stop >/dev/null 2>&1 - /sbin/chkconfig --del rhn-search -fi - -%files -%defattr(644,root,root,755) -%attr(755, root, root) %{_var}/log/rhn/search -%{_prefix}/share/rhn/search/lib/* -%{_prefix}/share/rhn/search/classes/log4j.properties -%{_prefix}/share/rhn/search/classes/com/* -%attr(755, root, root) %{_prefix}/share/rhn/search/indexes -%attr(755, root, root) %{_initrddir}/rhn-search -%attr(755, root, root) %{_bindir}/rhnsearchd -%dir %{_sysconfdir}/rhn/search/ -%config(noreplace) %{_sysconfdir}/rhn/search/rhn_search.conf -%config(noreplace) %{_sysconfdir}/rhn/search/rhn_search_daemon.conf -%{_sysconfdir}/logrotate.d/rhn-search - -%changelog -* Fri Jan 15 2010 Michael Mraka michael.mraka@redhat.com 0.8.1-1 -- rebuild for spacewalk 0.8 - -* Wed Dec 2 2009 Miroslav SuchÃœ msuchy@redhat.com 0.7.3-1 -- 480342 - move init script from /etc/init.d to /etc/rc.d/init.d - -* Wed Nov 25 2009 Miroslav SuchÃœ msuchy@redhat.com 0.7.1-1 -- 516872 - Fixed search server's log properties (paji@redhat.com) -- Update doc indexes to reside in "en-US" (jmatthew@redhat.com) -- bumping versions to 0.7.0 (jmatthew@redhat.com) - -* Thu Jun 25 2009 John Matthews jmatthew@redhat.com 0.6.11-1 -- 487014 - SystemSearch remove score requirement to redirect to SDC on 1 result - (jmatthew@redhat.com) -- DocumentationSearch fixing returned links so they are relative and update for - segment dir (jmatthew@redhat.com) - -* Mon Jun 01 2009 jesus m. rodriguez jesusr@redhat.com 0.6.10-1 -- added java debugger options for local dev config (jesusr@redhat.com) -- store pid file locally. (jesusr@redhat.com) -- 487014 - SystemSearch reducing scrore threshold for a single result to - redirect to SDC page (jmatthew@redhat.com) - -* Tue May 26 2009 Devan Goodwin dgoodwin@redhat.com 0.6.9-1 -- 501925 - "rhn-search cleanindex" will now restart search after cleaning index - (jmatthew@redhat.com) - -* Thu May 21 2009 jesus m. rodriguez jesusr@redhat.com 0.6.8-1 -- 457350 - added package search apis to match functionality webui provides (jmatthew@redhat.com) -- Fixing "free form" search. Adding a boolean flag which when passed (jmatthew@redhat.com) - -* Fri May 08 2009 jesus m. rodriguez jesusr@redhat.com 0.6.7-1 -- clean up javadoc in AdminHandler (jesusr@redhat.com) - -* Tue May 05 2009 jesus m. rodriguez jesusr@redhat.com 0.6.6-1 -- cleanup ScheduleManager, fix potential null pointer, added a testcase. - -* Wed Apr 22 2009 jesus m. rodriguez jesusr@redhat.com 0.6.5-1 -- Add /etc/rhn/search/ %dir directive in search spec. (dgoodwin@redhat.com) - -* Fri Apr 17 2009 Devan Goodwin dgoodwin@redhat.com 0.6.4-1 -- Search server adding a xmlrpc call that will allow updates to be triggered - immediately (jmatthew@redhat.com) -- 487209 - enhanced system search for hardware devices (jmatthew@redhat.com) -- 495921 - Limit results returned from Errata Search by Advisory - (jmatthew@redhat.com) -- 487158 - SystemSearch fixed search by customInfo (jmatthew@redhat.com) -- Search Server adding a debug option to print out explanation of searches - (jmatthew@redhat.com) -- 442439 - enhancing csv for systemsearch (jmatthew@redhat.com) -- 487189 - System Search fixed search by checkin (jmatthew@redhat.com) - -* Sun Apr 05 2009 jesus m. rodriguez jesusr@redhat.com 0.6.3-1 -- 487424 - add logrotate to %%files section (jesusr@redhat.com) - -* Sat Apr 04 2009 jesus m. rodriguez jesusr@redhat.com 0.6.2-1 -- install logrotate.d directory -- remove / after RPM_BUILD_ROOT it's not needed -- search requires doc-indexes, sw-doc-indexes provides doc-indexes (jesusr@redhat.com) -- 492624 - "rhn-search cleanindex" checks db connection is up (jmatthew@redhat.com) -- bump Versions to 0.6.0 (jesusr@redhat.com) - -* Tue Mar 31 2009 jesus m. rodriguez jesusr@redhat.com 0.5.10-1 -- 487424 - defined logrotate for rhn_search.log - -* Thu Mar 31 2009 jesus m. rodriguez jesusr@redhat.com -- 487424 - define logrotate for rhn_search.log - -* Thu Feb 26 2009 jesus m. rodriguez jesusr@redhat.com 0.5.9-1 -- systems query changed to no longer require anything more than rhnServer. - -* Thu Feb 19 2009 jesus m. rodriguez jesusr@redhat.com 0.5.8-1 -- Fixing problem which broke unique documents in the lucene index. -- SearchServer - SystemSearch lowering minimum score threshold - -* Thu Feb 19 2009 jesus m. rodriguez jesusr@redhat.com 0.5.7-1 -- 486182 - SystemSearch "id" is now tokenized, allows flexible 'ngram' s -- 485820 - System Search: Isn't reindexing when system data is modified - -* Thu Feb 12 2009 jesus m. rodriguez jesusr@redhat.com 0.5.6-1 -- 484610 - doc search results were reversed, displaying worst results first -- Cleaned up "tabs" which got added into some files previously. -- Removed "orig" as a field from DocResults - -* Thu Feb 05 2009 jesus m. rodriguez jesusr@redhat.com 0.5.5-1 -- 479541, 483867 - replaced runuser with /sbin/runuser - -* Mon Jan 26 2009 jesus m. rodriguez jesusr@redhat.com 0.5.4-1 -- allow search-server to run in degraded mode if nutch isn't installed -- fixing rpmlint warning: summary-ended-with-dot -- allow multiple language search - -* Fri Jan 23 2009 Devan Goodwin dgoodwin@redhat.com 0.5.3-1 -- Fix bad install command. - -* Thu Jan 22 2009 Dennis Gilmore dennis@ausil.us 0.5.2-1 -- update java-devel BuildRequires to 1.6.0 -- preserve timestamps on install - -* Wed Jan 21 2009 Milan Zazrivec mzazrivec@redhat.com 0.5.1-1 -- obsolete rhn-search before 5.3.0 (for satellite's benefit) - -* Tue Jan 13 2009 John Matthews jmatthews@redhat.com 0.4.10-1 -- Added SystemSearch parameter, running kernel - -* Thu Dec 18 2008 John Matthews jmatthews@redhat.com 0.4.7-1 -- rebuild for spacewalk 0.4 -- added doc search - -* Mon Dec 8 2008 John Matthews jmatthews@redhat.com 0.4.1-1 -- updates for "make test-srpm" to function - -* Fri Oct 24 2008 Jesus M. Rodriguez jesusr@redhat.com 0.3.4-1 -- rebuild -- added a "cleanindex" option to init.d script for rhn-search - -* Tue Oct 21 2008 Michael Mraka michael.mraka@redhat.com 0.3.3-1 -- resolves #467717 - fixed sysvinit scripts -- resolves #467877 - use runuser instead of su - -* Tue Sep 23 2008 Milan Zazrivec 0.3.1-1 -- fixed package obsoletes - -* Wed Sep 3 2008 Milan Zazrivec 0.2.6-1 -- config file needs to point to correct spacewalk-search.jar - -* Tue Sep 2 2008 Jesus Rodriguez 0.2.5-1 -- tagged for rebuild -- includes errata search capability -- fix setup and source0 to be name-version -- removed unnecessary bloat from libsrc directory -- removed apache-ibatis-sqlmap as a requires for now. FIXME - -* Mon Aug 11 2008 Jesus Rodriguez 0.1.2-1 -- tagged for rebuild after rename, also bumping version - -* Tue Aug 5 2008 Jan Pazdziora 0.1.2-0 -- tagged for rebuild after rename, also bumping version - -* Mon Aug 4 2008 Jan Pazdziora 0.1.1-0 -- rebuilt with BuildRequires: java-devel >= 1.5.0 diff --git a/search-server/spacewalk-search/Makefile b/search-server/spacewalk-search/Makefile new file mode 100644 index 0000000..700fc9e --- /dev/null +++ b/search-server/spacewalk-search/Makefile @@ -0,0 +1,2 @@ + +include ../rel-eng/Makefile diff --git a/search-server/spacewalk-search/README b/search-server/spacewalk-search/README new file mode 100644 index 0000000..b6f3d23 --- /dev/null +++ b/search-server/spacewalk-search/README @@ -0,0 +1,17 @@ +search-server is the new server used to search packages, and ultimately all +content including systems, errata, and documentation. + +build.properties - build configuration +build.xml - master ant build script +buildconf/exclude - list of files excluded from compilation (temp solution) +buildconf/build-utils.xml - ant macro definitions +checkstyle.xml - checkstyle code style definition +ivy.xml - ivy repo configuration file +libsrc - src code for the libraries used +nutch - configuration files for nutch crawler (not currently used) +spacewalk-search.spec - search server RPM spec file +rhn_search_daemon_dev.conf - development based configuration file +run.sh - script used to run the software, especially during development +scripts - useful scripts +src - the source code +task-lib - the libraries used by the search-server diff --git a/search-server/spacewalk-search/build.properties b/search-server/spacewalk-search/build.properties new file mode 100644 index 0000000..b8fd474 --- /dev/null +++ b/search-server/spacewalk-search/build.properties @@ -0,0 +1,22 @@ +java.src.dir=src/java +config.src.dir=src/config +java.lib.dir=lib +java.task.lib.dir=task-lib +report.dir=report +build.dir=build +build.instrument.dir=instrumented +dist.dir=dist +jar.name=spacewalk-search +jar.version=1.0 +jar.file.name=${jar.name}-${jar.version}.jar +ivyserver= http://jmrodri.fedorapeople.org +ivy.log.module.when.found=false +ivy.log.resolved.revision=false +# Checkstyle config entries +checkstyle.cache.file=.checkstyle_cache +checkstyle.header.file=./buildconf/LICENSE.txt +javadoc.method.scope=public +javadoc.type.scope=package +javadoc.var.scope=package +javadoc.lazy=true +jpackage.jars="commons-lang commons-logging redstone-xmlrpc redstone-xmlrpc-client quartz tanukiwrapper log4j commons-httpclient oro commons-cli commons-codec" diff --git a/search-server/spacewalk-search/build.xml b/search-server/spacewalk-search/build.xml new file mode 100644 index 0000000..de8e25f --- /dev/null +++ b/search-server/spacewalk-search/build.xml @@ -0,0 +1,241 @@ +<project name="search-server" basedir="." default="all"> + <import file="buildconf/build-utils.xml" /> + + <property file="build.properties" /> + + <path id="project.taskjars"> + <fileset dir="${java.task.lib.dir}" includes="**/*.jar" /> + </path> + + <path id="project.classpath"> + <fileset dir="${java.lib.dir}" includes="**/*.jar" /> + <fileset dir="${java.task.lib.dir}" > + <include name="junit*.jar" /> + <include name="emma*.jar" /> + </fileset> + </path> + + <path id="test.classpath"> + <pathelement location="${build.dir}" /> + <path refid="project.classpath" /> + <pathelement location="${basedir}/src/config" /> + </path> + + <target name="init-tasks"> + <taskdef name="ivy-retrieve" + classname="fr.jayasoft.ivy.ant.IvyRetrieve" + classpathref="project.taskjars"/> + <taskdef name="ivy-configure" + classname="fr.jayasoft.ivy.ant.IvyConfigure" + classpathref="project.taskjars"/> + <taskdef name="ivy-resolve" + classname="fr.jayasoft.ivy.ant.IvyResolve" + classpathref="project.taskjars"/> + <taskdef name="ivy-publish" + classname="fr.jayasoft.ivy.ant.IvyPublish" + classpathref="project.taskjars"/> + <taskdef resource="checkstyletask.properties"> + <classpath> + <pathelement path="${java.task.lib.dir}/checkstyle-4.1.jar" /> + <pathelement path="${java.task.lib.dir}/commons-beanutils-1.7.0.jar" /> + <pathelement path="${java.task.lib.dir}/commons-logging-1.0.4.jar" /> + <pathelement path="${java.task.lib.dir}/antlr-2.7.6.jar" /> + <pathelement path="${java.task.lib.dir}/regexp-1.3.jar" /> + </classpath> + </taskdef> + <taskdef resource="emma_ant.properties" classpathref="project.taskjars" /> + <taskdef resource="net/sf/antcontrib/antlib.xml"> + <classpath> + <pathelement location="${java.task.lib.dir}/ant-contrib-1.0.jar"/> + </classpath> + </taskdef> + </target> + + <target name="check-testcase-name" if="testcase" depends="init-tasks"> + <propertyregex property="valid.testcase" + input="${testcase}" + regexp="(.*)Test$" + select="\1" + casesensitive="true" /> + </target> + + <target name="check-testcase" depends="check-testcase-name" if="testcase" + unless="valid.testcase"> + <fail message="Invalid testcase name: ${testcase}. Class name must look like .*Test"/> + </target> + + <target name="test" depends="compile, check-testcase" + description="Run the tests (use -Dtestcase=<testcase>)"> + <mkdir dir="${report.dir}" /> + <!-- set testcase to wildcard "*" to test everything, unless + a specific testcase name was supplied. --> + <if> + <not> + <isset property="${testcase}" /> + </not> + <then> + <property name="testcase" value="*Test"/> + </then> + </if> + <junit> + <sysproperty key="rhn.config.dir" value="/etc/rhn" /> + <sysproperty key="search.config.dir" value="src/config/search" /> + <sysproperty key="log4j.configuration" value="log4j.test.properties" /> + <classpath refid="test.classpath" /> + <formatter type="brief" usefile="false" /> + <formatter type="xml" /> + <batchtest fork="yes" todir="${report.dir}"> + <fileset dir="${build.dir}" includes="**/${testcase}.class" /> + </batchtest> + </junit> + </target> + + <target name="checkstyle" depends="init-tasks, package"> + <delete dir="${report.dir}" /> + <mkdir dir="${report.dir}" /> + <checkstyle config="checkstyle.xml" failOnViolation="true" > + <property key="javadoc.method.scope" value="public" /> + <property key="javadoc.type.scope" value="package" /> + <property key="javadoc.var.scope" value="package" /> + <property key="javadoc.lazy" value="${javadoc.lazy}" /> + <fileset dir="${java.src.dir}" > + <include name="**/*.java" /> + <exclude name="**/tests/**" /> + </fileset> + <classpath> + <fileset dir="${java.lib.dir}" includes="**/*.jar" /> + <fileset dir="${java.task.lib.dir}" includes="junit*.jar" /> + <fileset dir="${dist.dir}" includes="${jar.file.name}" /> + </classpath> + <formatter type="plain" /> + <formatter type="xml" toFile="${report.dir}/checkstyle_report.xml" /> + </checkstyle> + </target> + + <target name="resolve-local" description="resolve jars via jpackage" if="installbuild"> + <copy toDir="${java.lib.dir}" overwrite="true"> + <fileset dir="buildconf/tempjars"> + <include name="**/*.jar" /> + </fileset> + </copy> + + <jpackage-deps jars="${jpackage.jars}" dir="${java.lib.dir}" /> + </target> + + <target name="resolve-ivy" description="retrieve dependencies with ivy" + depends="init-tasks" unless="installbuild"> + <!-- properties set here to show we are overriding default ivy values --> + <property name="ivy.default.ivy.user.dir" value="${user.home}/.ivy/search-server"/> + <property name="ivy.local.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> + <property name="ivy.ivyrep.default.ivy.root" value="${ivyserver}/ivy/" /> + <property name="ivy.ivyrep.default.artifact.root" value="${ivyserver}/ivy/" /> + <property name="ivy.ivyrep.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> + <ivy-retrieve/> + </target> + + <target name="resolve" depends="resolve-ivy,resolve-local" /> + + <target name="prep"> + <mkdir dir="${build.dir}" /> + <mkdir dir="${dist.dir}" /> + <mkdir dir="${java.lib.dir}" /> + </target> + + <target name="clean"> + <delete dir="${build.dir}" /> + <delete dir="${build.instrument.dir}" /> + <delete dir="${dist.dir}" /> + <delete dir="${java.lib.dir}" /> + <delete dir="${report.dir}" /> + <delete file="coverage.ec" /> + <delete file="rhn_search_daemon.log" /> + </target> + + <target name="compile" depends="prep, resolve"> + <javac srcdir="${java.src.dir}" + classpathref="project.classpath" + destdir="${build.dir}" + deprecation="on" + excludesfile="buildconf/exclude" + debug="on" /> + <copy todir="${build.dir}/com/redhat/satellite/search/db"> + <fileset dir="${config.src.dir}/com/redhat/satellite/search/db/"> + <include name="*.xml" /> + </fileset> + </copy> + <copy file="${config.src.dir}/log4j.properties" todir="${build.dir}" /> + </target> + + <target name="package" depends="compile"> + <jar basedir="${build.dir}" + destfile="${dist.dir}/${jar.file.name}" + excludes="**/tests/**" + includes="**/**" /> + </target> + + <target name="create-local-repo" description="Create a local Ivy repo"> + <mkdir dir="${user.home}/.ivy/search-server/local/" /> + </target> + + <target name="clean-cache" description="Removes the Ivy jar cache"> + <delete dir="${lib.dir}" quiet="true" /> + <delete dir="${user.home}/.ivy/search-server/cache/" quiet="true" /> + </target> + + <target name="all" depends="package" /> + + <target name="init-install"> + <property name="installbuild" value="true"/> + </target> + + <target name="install" depends="init-install,package" /> + + <target name="instrument" depends="init-tasks"> + <mkdir dir="${report.dir}" /> + <mkdir dir="${build.instrument.dir}" /> + + <emma enabled="true"> + <instr merge="no" + mode="overwrite" + instrpath="${build.dir}" + outfile="${report.dir}/coverage.em" > + <filter excludes="*Exception" /> + <filter excludes="*.tests.*" /> + </instr> + <!-- + <instr outdir="${build.instrument.dir}" merge="no" + instrpath="${build.dir}" + mode="fullcopy" + outfile="${report.dir}/coverage.em" > + <filter excludes="*Exception" /> + <filter excludes="*.tests.*" /> + </instr> + --> + </emma> + + <!-- + <copy toDir="${build.instrument.dir}/classes"> + <fileset dir="${build.dir}"> + <include name="**/*.xml" /> + </fileset> + </copy> + --> + </target> + + <target name="coverage-report" depends="instrument"> + <emma enabled="true"> + <report sourcepath="${java.src.dir}" + sort="+block,+name,+method,+class" + metrics="method:70,block:80,line:80,class:100"> + <fileset dir="${report.dir}" includes="*.em" /> + <fileset dir="." includes="*.ec" /> + <html outfile="${report.dir}/emma.html" + depth="method" + columns="name,class,method,block,line" /> + <xml outfile="${report.dir}/emma.xml" + depth="method" + columns="name,class,method,block,line" /> + </report> + </emma> + </target> +</project> diff --git a/search-server/spacewalk-search/buildconf/LICENSE.txt b/search-server/spacewalk-search/buildconf/LICENSE.txt new file mode 100644 index 0000000..629462a --- /dev/null +++ b/search-server/spacewalk-search/buildconf/LICENSE.txt @@ -0,0 +1,14 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ diff --git a/search-server/spacewalk-search/buildconf/build-utils.xml b/search-server/spacewalk-search/buildconf/build-utils.xml new file mode 100644 index 0000000..0011a83 --- /dev/null +++ b/search-server/spacewalk-search/buildconf/build-utils.xml @@ -0,0 +1,19 @@ +<project name="build-utils.xml"> + <!-- + creates symlinks for jpackage jars + requires: jpackage-utils + other: requires rpms of the jars specified to be installed + --> + <macrodef name="jpackage-deps"> + <attribute name="jars" /> + <attribute name="dir" /> + + <sequential> + <exec executable="build-jar-repository" failonerror="true" > + <arg value="--preserve-naming" /> + <arg value="-s @{dir}" /> + <arg line="@{jars}" /> + </exec> + </sequential> + </macrodef> +</project> diff --git a/search-server/spacewalk-search/buildconf/exclude b/search-server/spacewalk-search/buildconf/exclude new file mode 100644 index 0000000..7873251 --- /dev/null +++ b/search-server/spacewalk-search/buildconf/exclude @@ -0,0 +1,2 @@ +**/WebCrawl.java +**/IndexDocumentsTask.java diff --git a/search-server/spacewalk-search/buildconf/tempjars/hadoop-0.18.1-core.jar b/search-server/spacewalk-search/buildconf/tempjars/hadoop-0.18.1-core.jar new file mode 100644 index 0000000..a024c3c Binary files /dev/null and b/search-server/spacewalk-search/buildconf/tempjars/hadoop-0.18.1-core.jar differ diff --git a/search-server/spacewalk-search/buildconf/tempjars/ibatis-2.3.0.677.jar b/search-server/spacewalk-search/buildconf/tempjars/ibatis-2.3.0.677.jar new file mode 100644 index 0000000..72dbe19 Binary files /dev/null and b/search-server/spacewalk-search/buildconf/tempjars/ibatis-2.3.0.677.jar differ diff --git a/search-server/spacewalk-search/buildconf/tempjars/lucene-analyzers-2.3.2.jar b/search-server/spacewalk-search/buildconf/tempjars/lucene-analyzers-2.3.2.jar new file mode 100644 index 0000000..cf0d296 Binary files /dev/null and b/search-server/spacewalk-search/buildconf/tempjars/lucene-analyzers-2.3.2.jar differ diff --git a/search-server/spacewalk-search/buildconf/tempjars/lucene-core-2.3.2.jar b/search-server/spacewalk-search/buildconf/tempjars/lucene-core-2.3.2.jar new file mode 100644 index 0000000..639f750 Binary files /dev/null and b/search-server/spacewalk-search/buildconf/tempjars/lucene-core-2.3.2.jar differ diff --git a/search-server/spacewalk-search/buildconf/tempjars/nutch-2008-12-01_04-01-21.jar b/search-server/spacewalk-search/buildconf/tempjars/nutch-2008-12-01_04-01-21.jar new file mode 100644 index 0000000..4bdec09 Binary files /dev/null and b/search-server/spacewalk-search/buildconf/tempjars/nutch-2008-12-01_04-01-21.jar differ diff --git a/search-server/spacewalk-search/buildconf/tempjars/picocontainer-1.3.jar b/search-server/spacewalk-search/buildconf/tempjars/picocontainer-1.3.jar new file mode 100644 index 0000000..9a3f670 Binary files /dev/null and b/search-server/spacewalk-search/buildconf/tempjars/picocontainer-1.3.jar differ diff --git a/search-server/spacewalk-search/buildconf/tempjars/simple-core-3.1.3.jar b/search-server/spacewalk-search/buildconf/tempjars/simple-core-3.1.3.jar new file mode 100644 index 0000000..80b44c1 Binary files /dev/null and b/search-server/spacewalk-search/buildconf/tempjars/simple-core-3.1.3.jar differ diff --git a/search-server/spacewalk-search/checkstyle.xml b/search-server/spacewalk-search/checkstyle.xml new file mode 100644 index 0000000..89cde80 --- /dev/null +++ b/search-server/spacewalk-search/checkstyle.xml @@ -0,0 +1,233 @@ +<?xml version="1.0"?> +<!DOCTYPE module PUBLIC + "-//Puppy Crawl//DTD Check Configuration 1.1//EN" + "http://www.puppycrawl.com/dtds/configuration_1_1.dtd"> + +<!-- + + Checkstyle configuration that checks the sun coding conventions from: + + - the Java Language Specification at + http://java.sun.com/docs/books/jls/second_edition/html/index.html + + - the Sun Code Conventions at http://java.sun.com/docs/codeconv/ + + - the Javadoc guidelines at + http://java.sun.com/j2se/javadoc/writingdoccomments/index.html + + - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html + + - some best practices + + Checkstyle is very configurable. Be sure to read the documentation at + http://checkstyle.sf.net (or in your downloaded distribution). + + Most Checks are configurable, be sure to consult the documentation. + + To completely disable a check, just comment it out or delete it from the file. + + Finally, it is worth reading the documentation. + +--> + +<module name="Checker"> + + <!-- Checks that a package.html file exists for each package. --> + <!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml --> +<!-- <module name="PackageHtml"/> --> + + <!-- Checks whether files end with a new line. --> + <!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile --> + <module name="NewlineAtEndOfFile"/> + + <!-- Checks that property files contain the same keys. --> + <!-- See http://checkstyle.sf.net/config_misc.html#Translation --> + <module name="Translation"/> + + <module name="TreeWalker"> + + <property name="cacheFile" value="${checkstyle.cache.file}"/> + + <!-- ************************************************************** --> + <!-- Checks that are different from the sun coding conventions ones --> + <!-- ************************************************************** --> + <property name="tabWidth" value="4"/> + <module name="LeftCurly" /> + <module name="RightCurly"> + <property name="option" value="alone"/> + </module> + <module name="LineLength"> + <property name="ignorePattern" value="import"/> + <property name="max" value="92"/> + </module> + + <!-- ************************************************************** --> + <!-- Default Sun coding conventions checks --> + <!-- ************************************************************** --> + + <!-- Checks for Javadoc comments. --> + <!-- See http://checkstyle.sf.net/config_javadoc.html --> + <module name="JavadocMethod"> + <property name="scope" value="${javadoc.method.scope}" /> + <property name="allowUndeclaredRTE" value="true" /> + <property name="allowThrowsTagsForSubclasses" value="${javadoc.lazy}" /> + <property name="allowMissingParamTags" value="${javadoc.lazy}" /> + <property name="allowMissingThrowsTags" value="${javadoc.lazy}" /> + <property name="allowMissingReturnTag" value="${javadoc.lazy}" /> + </module> + <module name="JavadocType"> + <property name="versionFormat" value="\$Rev.*\$"/> + <property name="scope" value="${javadoc.type.scope}" /> + </module> + <module name="JavadocVariable"> + <property name="scope" value="${javadoc.var.scope}" /> + <property name="severity" value="ignore"/> + </module> + +<!-- JavadocStyle not supported by some versions. --> +<!-- <module name="JavadocStyle"/> --> + + + <!-- Checks for Naming Conventions. --> + <!-- See http://checkstyle.sf.net/config_naming.html --> + <module name="ConstantName"/> + <module name="LocalFinalVariableName"/> + <module name="LocalVariableName"/> + <module name="MethodName"/> + <module name="PackageName"> + <property name="format" value="^[a-z]+(\.[A-Za-z][A-Za-z0-9]*)*$"/> + </module> + <module name="ParameterName"/> + <module name="StaticVariableName"/> + <module name="TypeName"/> + + + <!-- Checks for Headers --> + <!-- See http://checkstyle.sf.net/config_header.html --> + <module name="Header"> + <!-- The follow property value demonstrates the ability --> + <!-- to have access to ANT properties. In this case it uses --> + <!-- the ${basedir} property to allow Checkstyle to be run --> + <!-- from any directory within a project. --> + <property name="headerFile" value="${checkstyle.header.file}"/> + </module> + + <!-- Following interprets the header file as regular expressions. --> + <!-- <module name="RegexpHeader"/> --> + + + + <!-- Checks for imports --> + <!-- See http://checkstyle.sf.net/config_import.html --> + <module name="AvoidStarImport"/> + <module name="IllegalImport"/> <!-- defaults to sun.* packages --> + <module name="RedundantImport"/> + <module name="UnusedImports"/> + + + <!-- Checks for Size Violations. --> + <!-- See http://checkstyle.sf.net/config_sizes.html --> + <module name="MethodLength"> + <property name="tokens" value="METHOD_DEF"/> + <property name="max" value="150"/> + <property name="countEmpty" value="false"/> + </module> + <module name="ParameterNumber"/> + + + <!-- Checks for whitespace --> + <!-- See http://checkstyle.sf.net/config_whitespace.html --> + <module name="EmptyForIteratorPad"/> + <module name="NoWhitespaceAfter"> + <property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, UNARY_PLUS, TYPECAST, GENERIC_START"/> + </module> + <module name="NoWhitespaceBefore"> + <property name="tokens" value="SEMI, POST_DEC, POST_INC, GENERIC_START, GENERIC_END"/> + </module> + <module name="OperatorWrap"> + <property name="option" value="eol" /> + </module> + <module name="ParenPad"/> +<!-- <module name="TypecastParenPad"/> --> + <module name="TabCharacter"/> + <module name="WhitespaceAfter"> + <property name="tokens" value="COMMA, SEMI, GENERIC_END"/> + </module> + <module name="WhitespaceAround"> + <property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, +COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LCURLY, LE, LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, +LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS, +MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION, RCURLY, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN, STAR, STAR_ASSIGN, TYPE_EXTENSION_AND, WILDCARD_TYPE"/> + </module> + <module name="MethodParamPad"> + <property name="allowLineBreaks" value="true"/> + </module> + + + <!-- Modifier Checks --> + <!-- See http://checkstyle.sf.net/config_modifiers.html --> + <module name="ModifierOrder"/> + <module name="RedundantModifier"/> + + + <!-- Checks for blocks. You know, those {}'s --> + <!-- See http://checkstyle.sf.net/config_blocks.html --> + <module name="AvoidNestedBlocks"/> + <module name="EmptyBlock"> + <property name="option" value="text"/> + </module> + + <module name="NeedBraces"/> + + + <!-- Checks for common coding problems --> + <!-- See http://checkstyle.sf.net/config_coding.html --> + <!-- <module name="AvoidInlineConditionals"/> --> +<!-- <module name="CovariantEquals"/> --> + <module name="DoubleCheckedLocking"/> <!-- MY FAVOURITE --> + <module name="EmptyStatement"/> + <module name="EqualsHashCode"/> + <module name="HiddenField"/> + <module name="IllegalInstantiation"/> + <module name="InnerAssignment"/> + <!-- <module name="MagicNumber" /> --> + <module name="MissingSwitchDefault"/> +<!-- <module name="RedundantThrows"/> --> + <module name="SimplifyBooleanExpression"/> + <module name="SimplifyBooleanReturn"/> +<!-- <module name="IllegalCatch"/> --> +<!-- <module name="PackageDeclaration"/> --> + + <!-- Checks for class design --> + <!-- See http://checkstyle.sf.net/config_design.html --> +<!-- <module name="DesignForExtension"/>--> +<!-- <module name="FinalClass"/> --> + <module name="HideUtilityClassConstructor"/> + <module name="InterfaceIsType"/> + <module name="VisibilityModifier"> + <property name="protectedAllowed" value="true" /> + </module> + + + <!-- Miscellaneous other checks. --> + <!-- See http://checkstyle.sf.net/config_misc.html --> + <module name="ArrayTypeStyle"/> +<!-- <module name="FinalParameters"/> --> +<!-- + <module name="GenericIllegalRegexp"> + <property name="format" value="System\.out\.println"/> + </module> +--> + <module name="UpperEll"/> + + <!-- Optional Checks --> + <!-- (gack, not available until version 3 of checkstyle --> +<!-- + <module name="usage.OneMethodPrivateField"/> + <module name="usage.UnusedLocalVariable"/> + <module name="usage.UnusedParameter"/> + <module name="usage.UnusedPrivateField"/> +--> + </module> + +</module> diff --git a/search-server/spacewalk-search/ivy.xml b/search-server/spacewalk-search/ivy.xml new file mode 100644 index 0000000..aec44ca --- /dev/null +++ b/search-server/spacewalk-search/ivy.xml @@ -0,0 +1,28 @@ +<ivy-module version="1.0"> + <info organisation="redhat" module="rhn-java" /> + <dependencies> + <dependency org="redhat" name="simple-core" rev="3.1.3" /> + <dependency org="redhat" name="lucene-core" rev="2.3.2" /> + <dependency org="redhat" name="lucene-analyzers" rev="2.3.2" /> + <dependency org="redhat" name="commons-lang" rev="2.0" /> + <dependency org="redhat" name="commons-logging" rev="1.1" /> + <dependency org="redhat" name="redstone-xmlrpc" rev="1.1_20071120" /> + <dependency org="redhat" name="redstone-xmlrpc-client" rev="1.1_20071120" /> + <dependency org="redhat" name="ibatis" rev="2.3.0.677" /> + <dependency org="redhat" name="ojdbc14" rev="1.10" /> + <dependency org="redhat" name="quartz" rev="1.5.1" /> + <dependency org="redhat" name="picocontainer" rev="1.3" /> + <dependency org="redhat" name="tanukiwrapper" rev="3.1.2" /> + <dependency org="redhat" name="log4j" rev="1.2.8" /> + <dependency org="redhat" name="logdriver" rev="0.1" /> + <dependency org="redhat" name="nutch" rev="2008-12-01_04-01-21" /> + <dependency org="redhat" name="hadoop" rev="0.18.1-core" /> + <!-- dependency org="redhat" name="commons-httpclient" rev="3.0" / --> + <!-- dependency org="redhat" name="oro" rev="2.0.8" / --> + <!-- dependency org="redhat" name="commons-cli" rev="1.0" / --> + <!-- dependency org="redhat" name="commons-codec" rev="1.2" / --> + <!-- dependency org="redhat" name="tika" rev="0.1-incubating" / --> + <!-- dependency org="redhat" name="icu4j" rev="3_8_1" / --> + <!-- dependency org="redhat" name="icu4j-charsets" rev="3_8_1" / --> + </dependencies> +</ivy-module> diff --git a/search-server/spacewalk-search/libsrc/hadoop-0.16.4-src.jar b/search-server/spacewalk-search/libsrc/hadoop-0.16.4-src.jar new file mode 100644 index 0000000..c941a7c Binary files /dev/null and b/search-server/spacewalk-search/libsrc/hadoop-0.16.4-src.jar differ diff --git a/search-server/spacewalk-search/libsrc/ibatis-2.3.0.677-src.zip b/search-server/spacewalk-search/libsrc/ibatis-2.3.0.677-src.zip new file mode 100644 index 0000000..8c3274b Binary files /dev/null and b/search-server/spacewalk-search/libsrc/ibatis-2.3.0.677-src.zip differ diff --git a/search-server/spacewalk-search/libsrc/icu4j-3_8_1-src.jar b/search-server/spacewalk-search/libsrc/icu4j-3_8_1-src.jar new file mode 100644 index 0000000..19dd665 Binary files /dev/null and b/search-server/spacewalk-search/libsrc/icu4j-3_8_1-src.jar differ diff --git a/search-server/spacewalk-search/libsrc/log4j-1.2.8-src.jar b/search-server/spacewalk-search/libsrc/log4j-1.2.8-src.jar new file mode 100644 index 0000000..b36df55 Binary files /dev/null and b/search-server/spacewalk-search/libsrc/log4j-1.2.8-src.jar differ diff --git a/search-server/spacewalk-search/libsrc/lucene-2.3.2-src.zip b/search-server/spacewalk-search/libsrc/lucene-2.3.2-src.zip new file mode 100644 index 0000000..b49b9b6 Binary files /dev/null and b/search-server/spacewalk-search/libsrc/lucene-2.3.2-src.zip differ diff --git a/search-server/spacewalk-search/libsrc/nutch-1.0dev_2008_05_14-src.jar b/search-server/spacewalk-search/libsrc/nutch-1.0dev_2008_05_14-src.jar new file mode 100644 index 0000000..372eca5 Binary files /dev/null and b/search-server/spacewalk-search/libsrc/nutch-1.0dev_2008_05_14-src.jar differ diff --git a/search-server/spacewalk-search/libsrc/nutch-2008-12-01_04-01-21-src.zip b/search-server/spacewalk-search/libsrc/nutch-2008-12-01_04-01-21-src.zip new file mode 100644 index 0000000..268a010 Binary files /dev/null and b/search-server/spacewalk-search/libsrc/nutch-2008-12-01_04-01-21-src.zip differ diff --git a/search-server/spacewalk-search/libsrc/quartz-1.5.1-src.jar b/search-server/spacewalk-search/libsrc/quartz-1.5.1-src.jar new file mode 100644 index 0000000..79edf1a Binary files /dev/null and b/search-server/spacewalk-search/libsrc/quartz-1.5.1-src.jar differ diff --git a/search-server/spacewalk-search/libsrc/redstone-xmlrpc-1.1_20071120-src.jar b/search-server/spacewalk-search/libsrc/redstone-xmlrpc-1.1_20071120-src.jar new file mode 100644 index 0000000..2dc7cf7 Binary files /dev/null and b/search-server/spacewalk-search/libsrc/redstone-xmlrpc-1.1_20071120-src.jar differ diff --git a/search-server/spacewalk-search/libsrc/simple-3.1.3-src.zip b/search-server/spacewalk-search/libsrc/simple-3.1.3-src.zip new file mode 100644 index 0000000..cdd83c6 Binary files /dev/null and b/search-server/spacewalk-search/libsrc/simple-3.1.3-src.zip differ diff --git a/search-server/spacewalk-search/libsrc/tanukiwrapper-3.1.2-src.jar b/search-server/spacewalk-search/libsrc/tanukiwrapper-3.1.2-src.jar new file mode 100644 index 0000000..9436fbd Binary files /dev/null and b/search-server/spacewalk-search/libsrc/tanukiwrapper-3.1.2-src.jar differ diff --git a/search-server/spacewalk-search/nutch/crawl_jsp.sh b/search-server/spacewalk-search/nutch/crawl_jsp.sh new file mode 100755 index 0000000..d7a7e19 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp.sh @@ -0,0 +1,35 @@ +#!/bin/bash + + +if [ "${JAVA_HOME}" = "" ]; then + export JAVA_HOME=/usr/lib/jvm/java + echo "Setting JAVA_HOME to: ${JAVA_HOME}" +fi + +export NUTCH_HOME=/usr/share/nutch +export NUTCH_CONF_DIR=`pwd`/crawl_jsp/conf +export NUTCH_OPTS=-Djava.util.logging.config.file="${NUTCH_CONF}/logging.properties" +export NUTCH_LOG_DIR=`pwd`/logs +export NUTCH_LOG_FILE=$0.log +export OUTPUT_DIR=`pwd`/data/crawl_jsp + +echo "NUTCH_HOME = ${NUTCH_HOME}" +echo "NUTCH_CONF_DIR = ${NUTCH_CONF_DIR}" +echo "NUTCH_OPTS = ${NUTCH_OPTS}" +echo "NUTCH_LOG_DIR = ${NUTCH_LOG_DIR}" +echo "NUTCH_LOG_FILE = ${NUTCH_LOG_FILE}" +echo "OUTPUT_DIR = ${OUTPUT_DIR}" + +if [ ! -d ${NUTCH_LOG_DIR} ]; then + echo "Creating log directory ${NUTCH_LOG_DIR}" + mkdir ${NUTCH_LOG_DIR} +fi + + +if [ ! -d ${OUTPUT_DIR} ]; then + echo "Creating output directory ${OUTPUT_DIR}" + mkdir -p ${OUTPUT_DIR} +fi +#Need to adjust nutch RPM so it's scripts under bin are executable by all + +${NUTCH_HOME}/bin/nutch crawl ${NUTCH_CONF_DIR}/../urls -dir ${OUTPUT_DIR} -depth 10 -threads 50 diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/automaton-urlfilter.txt b/search-server/spacewalk-search/nutch/crawl_jsp/conf/automaton-urlfilter.txt new file mode 100644 index 0000000..84cf8fa --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/automaton-urlfilter.txt @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The default url filter. +# Better for whole-internet crawling. + +# Each non-comment, non-blank line contains a regular expression +# prefixed by '+' or '-'. The first matching pattern in the file +# determines whether a URL is included or ignored. If no pattern +# matches, the URL is ignored. + +# skip file: ftp: and mailto: urls +-(file|ftp|mailto):.* + +# skip image and other suffixes we can't yet parse +-.*.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe) + +# skip URLs containing certain characters as probable queries, etc. +-.*[?*!@=].* + +# accept anything else ++.* diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/common-terms.utf8 b/search-server/spacewalk-search/nutch/crawl_jsp/conf/common-terms.utf8 new file mode 100644 index 0000000..220d1da --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/common-terms.utf8 @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Common terms and phrases which will be indexed in n-grams +# in order to optimize search. +content:a +content:and +content:for +content:in +content:of +content:the +content:to +url:com +url:http +url:http-www +url:www diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/configuration.xsl b/search-server/spacewalk-search/nutch/crawl_jsp/conf/configuration.xsl new file mode 100644 index 0000000..79141dc --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/configuration.xsl @@ -0,0 +1,40 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> +<xsl:output method="html"/> +<xsl:template match="configuration"> +<html> +<body> +<table border="1"> +<tr> + <td>name</td> + <td>value</td> + <td>description</td> +</tr> +<xsl:for-each select="property"> +<tr> + <td><a name="{name}"><xsl:value-of select="name"/></a></td> + <td><xsl:value-of select="value"/></td> + <td><xsl:value-of select="description"/></td> +</tr> +</xsl:for-each> +</table> +</body> +</html> +</xsl:template> +</xsl:stylesheet> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/context.xsl b/search-server/spacewalk-search/nutch/crawl_jsp/conf/context.xsl new file mode 100644 index 0000000..fbb3bc6 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/context.xsl @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="ISO-8859-1"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author : Chris Mattmann + Auhtor : Jérôme Charron + Description: This xsl file is used to transform dynamic properties out of the + nutch-default.xml file into a deployable Context.xml file for configuring + the Nutch war file in a servlet container. +--> + +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform%22%3E + +<xsl:template match="/"> +<Context path="/nutch" docBase="nutch-0.9-dev.war" + debug="5" reloadable="true" crossContext="true"> + +<xsl:for-each select="configuration/property"> + +<!-- "searcher." properties --> +<xsl:call-template name="parameter"> + <xsl:with-param name="property" select="current()"/> + <xsl:with-param name="filter" select="'searcher.'"/> +</xsl:call-template> + +<!-- "plugin." properties --> +<xsl:call-template name="parameter"> + <xsl:with-param name="property" select="current()"/> + <xsl:with-param name="filter" select="'plugin.'"/> +</xsl:call-template> + +<!-- "extension.clustering." properties --> +<xsl:call-template name="parameter"> + <xsl:with-param name="property" select="current()"/> + <xsl:with-param name="filter" select="'extension.clustering.'"/> +</xsl:call-template> + +<!-- "extension.ontology." properties --> +<xsl:call-template name="parameter"> + <xsl:with-param name="property" select="current()"/> + <xsl:with-param name="filter" select="'extension.ontology.'"/> +</xsl:call-template> + +<!-- "query." properties --> +<xsl:call-template name="parameter"> + <xsl:with-param name="property" select="current()"/> + <xsl:with-param name="filter" select="'query.'"/> +</xsl:call-template> + +</xsl:for-each> + +</Context> +</xsl:template> + + +<!-- + ! Template used to write out a parameter if the property's + ! name contains the specified filter string. + !--> +<xsl:template name="parameter"> + <xsl:param name="property"/> + <xsl:param name="filter"/> + <xsl:if test="contains(name, $filter)"> + <Parameter override="false"> + <xsl:attribute name="name"> + <xsl:value-of select="name"/> + </xsl:attribute> + <xsl:attribute name="value"> + <xsl:value-of select="value"/> + </xsl:attribute> + </Parameter> + </xsl:if> +</xsl:template> + +</xsl:stylesheet> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-tool.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-tool.xml new file mode 100644 index 0000000..9f6d667 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-tool.xml @@ -0,0 +1,59 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Overidden defaults for intranet use. --> + +<!-- Do not modify this file directly. Instead, copy entries that you --> +<!-- wish to modify from this file into nutch-site.xml and change them --> +<!-- there. If nutch-site.xml does not already exist, create it. --> + +<configuration> + +<property> + <name>urlfilter.regex.file</name> + <value>crawl-urlfilter.txt</value> +</property> + +<property> + <name>db.ignore.internal.links</name> + <value>false</value> + <description>If true, when adding new links to a page, links from + the same host are ignored. This is an effective way to limit the + size of the link database, keeping the only the highest quality + links. + </description> +</property> + +<property> + <name>fetcher.server.delay</name> + <value>1.0</value> + <description>The number of seconds the fetcher will delay between + successive requests to the same server.</description> +</property> + +<property> + <name>http.max.delays</name> + <value>1000</value> + <description>The number of times a thread will delay when trying to + fetch a page. When using the crawl tool there are likely to be very + few different hosts, so we need to be willing to wait longer for + each.</description> +</property> + +</configuration> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-urlfilter.txt b/search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-urlfilter.txt new file mode 100644 index 0000000..2698c17 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/crawl-urlfilter.txt @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The url filter file used by the crawl command. + +# Better for intranet crawling. +# Be sure to change MY.DOMAIN.NAME to your domain name. + +# Each non-comment, non-blank line contains a regular expression +# prefixed by '+' or '-'. The first matching pattern in the file +# determines whether a URL is included or ignored. If no pattern +# matches, the URL is ignored. + +# skip file:, ftp:, & mailto: urls +#-^(file|ftp|mailto): +-^(http|ftp|mailto): + +# skip image and other suffixes we can't yet parse +-.(gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|jpeg|JPEG|bmp|BMP|pdf)$ + +# skip URLs containing certain characters as probable queries, etc. +-[?*!@=] + +# skip URLs with slash-delimited segment that repeats 3+ times, to break loops +-.*(/[^/]+)/[^/]+\1/[^/]+\1/ + +# accept hosts in MY.DOMAIN.NAME +#+^http://(%5Ba-z0-9%5D*%5C.)*MY.DOMAIN.NAME/ +#+^http://(%5Ba-z0-9%5D*%5C.)*redhat.com/ + +# skip everything else +#- + + +#accept everything else +#+.* + +#ingore all svn entries +-.*.svn +#only read in english docs from these locations ++.*/docs/guides/channel-mgmt/en/ ++.*/docs/guides/client-config/en/ ++.*/docs/guides/proxy/rhn511/en/ ++.*/docs/release-notes/satellite/rhn510/en-US/ ++.*/docs/guides/satellite/en/ ++.*/docs/guides/reference/en/ +#skip everything else +-.* diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xml new file mode 100644 index 0000000..635f5a2 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xml @@ -0,0 +1,4354 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Document : domain-suffixes.xml + Author : Enis Soztutar - enis.soz.nutch@gmail.com + Description: This document contains top level domains + as described by the Internet Assigned Numbers + Authotiry (IANA), and second or third level domains that + are known to be managed by domain registerers. People at + Mozilla community call these public suffixes or effective + tlds. There is no algorithmic way of knowing whether a suffix + is a public domain suffix, or not. So this large file is used + for this purpose. The entries in the file is used to find the + domain of a url, which may not the same thing as the host of + the url. For example for "http://lucene.apache.org/nutch" the + hostname is lucene.apache.org, however the domain name for this + url would be apache.org. Domain names can be quite handy for + statistical analysis, and fighting against spam. + + The list of TLDs is constructed from IANA, and the + list of "effective tlds" are constructed from Wikipedia, + http://wiki.mozilla.org/TLD_List, and http://publicsuffix.org/ + The list may not include all the suffixes, but some + effort has been spent to make it comprehensive. Please forward + any improvements for this list to nutch-dev mailing list, or + nutch JIRA. + + Top level domains(tlds) are grouped + to three, namely infrastrusture, generic and country + code tlds. Infrastrusture tlds are only used for + technical reasons. Generic tlds represents the type + of the organization that they represent. Those in + current use and those waiting for approval is listed. + Most of the country code tlds correspond to the two + letter ISO-3166-1 country codes. + Each tld is listed with its domain (such as com), a + status enumeration describing the status of the tld, + and optionally a description or description for convenience. + cctlds are listed with additional country name field. + + status and boost elements are optional, with default values IN_USE + and 1.0 respectively. see domain-suffixes.xsd for the xml schema. + + + References : + http://www.iana.org + http://www.iana.org/gtld/gtld.htm + http://www.iana.org/root-whois/index.html + http://en.wikipedia.org/wiki/Top-level_domain + http://en.wikipedia.org/wiki/List_of_Internet_top-level_domains + http://wiki.mozilla.org/TLD_List + http://publicsuffix.org/ + https://bugzilla.mozilla.org/show_bug.cgi?id=331510 + http://www.neuhaus.com/domaincheck/domain_list.htm +--> + +<domains xmlns="http://lucene.apache.org/nutch" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://lucene.apache.org/nutch domain-suffixes.xsd"> + + <tlds> + <!-- Infrastructure Top Level Domains --> + <itlds> + <tld domain="root"> + <status>INFRASTRUCTURE</status> + <description> + (from http://en.wikipedia.org/wiki/.root) + vrsn-end-of-zone-marker-dummy-record.root is a domain name + listed in the DNS root zone as a diagnostic marker, whose + presence demonstrates the root zone was not truncated upon + loading by a root nameserver. It could be argued it represents + a top-level domain of .root, although technically no such + delegation exists. + </description> + </tld> + + <tld domain="arpa"> + <status>INFRASTRUCTURE</status> + <description> + (from http://en.wikipedia.org/wiki/.arpa) .arpa is an Internet + top-level domain (TLD) used exclusively for + Internet-infrastructure purposes. It does not function as a + normal TLD where websites are registered, but rather as a + meta-TLD used to look up addresses, and for other purposes. + </description> + </tld> + </itlds><!-- Generic Top Level Domains --> + <gtlds> + <!-- + The following gTLDs are in actual use + --> + + <tld domain="aero"> + <status>SPONSORED</status> + <description>for the air transport industry</description> + </tld> + + <tld domain="biz"> + <status>UNSPONSORED</status> + <description>for business use</description> + </tld> + + <tld domain="cat"> + <status>SPONSORED</status> + <description>for Catalan language/culture</description> + </tld> + + <tld domain="com"> + <status>UNSPONSORED</status> + <description> + for commercial organizations, but unrestricted + </description> + </tld> + + <tld domain="coop"> + <status>SPONSORED</status> + <description>for cooperatives</description> + </tld> + + <tld domain="edu"> + <status>UNSPONSORED</status> + <boost>1.0</boost> + <description> + for post-secondary educational establishments + </description> + </tld> + + <tld domain="gov"> + <status>UNSPONSORED</status> + <description> + for governments and their agencies in the United States + </description> + </tld> + + <tld domain="info"> + <status>UNSPONSORED</status> + <description> + for informational sites, but unrestricted + </description> + </tld> + + <tld domain="int"> + <status>UNSPONSORED</status> + <description> + for international organizations established by treaty + </description> + </tld> + + <tld domain="jobs"> + <status>SPONSORED</status> + <description>for employment-related sites</description> + </tld> + + <tld domain="mil"> + <status>UNSPONSORED</status> + <description>for the US military</description> + </tld> + + <tld domain="mobi"> + <status>SPONSORED</status> + <description>for sites catering to mobile devices</description> + </tld> + + <tld domain="museum"> + <status>SPONSORED</status> + <description>for museums</description> + </tld> + + <tld domain="name"> + <status>UNSPONSORED</status> + <description>for families and individuals</description> + </tld> + + <tld domain="net"> + <status>UNSPONSORED</status> + <description> + originally for network infrastructures, now unrestricted + </description> + </tld> + + <tld domain="org"> + <status>UNSPONSORED</status> + <description> + originally for organizations not clearly falling within the + other gTLDs, now unrestricted + </description> + </tld> + + <tld domain="pro"> + <status>SPONSORED</status> + <description>for certain professions</description> + </tld> + + <tld domain="travel"> + <status>SPONSORED</status> + <description> + for travel agents, airlines, hoteliers, tourism bureaus, etc. + </description> + </tld> + + <!-- + The following gTLDs are in the process of being approved, + and may be added to the root nameservers in the near future + --> + + <tld domain="asia"> + <status>STARTUP</status> + <description>for the Asian community</description> + </tld> + + <tld domain="post"> + <status>PROPOSED</status> + <description>for postal services</description> + </tld> + + <tld domain="tel"> + <status>STARTUP</status> + <description> + for services involving connections between the telephone + network and the Internet + </description> + </tld> + + <tld domain="geo"> + <status>PROPOSED</status> + <description>for geographically related sites</description> + </tld> + + <tld domain="gal"> + <status>PROPOSED</status> + <description>for Galicia, a country within Spain</description> + </tld> + + <tld domain="cym"> + <status>PROPOSED</status> + <description>for Wales, a country within the UK</description> + </tld> + + <tld domain="sco"> + <status>PROPOSED</status> + <description>for Scotland, a country within the UK</description> + </tld> + + <tld domain="kid"> + <status>PROPOSED</status> + <description>for websites designed for children</description> + </tld> + + <tld domain="kids"> + <status>PROPOSED</status> + <description>for websites designed for children</description> + </tld> + + <tld domain="mail"> + <status>PROPOSED</status> + <description>http://en.wikipedia.org/wiki/.mail</description> + </tld> + + <tld domain="web"> + <status>PROPOSED</status> + <description>For Web sites of all sorts</description> + </tld> + + <tld domain="xxx"> + <status>PROPOSED</status> + <description>For Adult entertainment sites</description> + </tld> + + <!-- + The following gTLDs are removed from the registry + --> + <tld domain="nato"> + <status>DELETED</status> + <description> + for NATO sites and operations. Replaced by .int + </description> + </tld> + + <!-- + The following gTLDs are PSEUDO_DOMAINs + --> + <tld domain="bitnet"> + <status>PSEUDO_DOMAIN</status> + <description> + identifying a hostname not connected directly to the Internet, + but a bitnet network + </description> + </tld> + + <tld domain="csnet"> + <status>PSEUDO_DOMAIN</status> + <description> + identifying a hostname not connected directly to the Internet, + but a csnet network + </description> + </tld> + + <tld domain="uucp"> + <status>PSEUDO_DOMAIN</status> + <description> + identifying a hostname not connected directly to the Internet, + but a bitnet network + </description> + </tld> + + <tld domain="local"> + <status>PSEUDO_DOMAIN</status> + <description> + .local is a pseudo top-level domain used by Apple, Inc.'s + Bonjour protocol. + </description> + </tld> + + <tld domain="internal"> + <status>PSEUDO_DOMAIN</status> + <description>alias of .local</description> + </tld> + + <tld domain="onion"> + <status>PSEUDO_DOMAIN</status> + <description> + designates an anonymous or pseudonymous address reachable via + the Tor network. + </description> + </tld> + </gtlds><!-- Country Code Top Level Domains --> + + <cctlds> + <tld domain="ac"> + <country>Ascension Island</country> + </tld> + + <tld domain="ad"> + <country>Andorra</country> + </tld> + + <tld domain="ae"> + <country>United Arab Emirates</country> + </tld> + + <tld domain="af"> + <country>Afghanistan</country> + </tld> + + <tld domain="ag"> + <country>Antigua and Barbuda</country> + </tld> + + <tld domain="ai"> + <country>Anguilla</country> + </tld> + + <tld domain="al"> + <country>Albania</country> + </tld> + + <tld domain="am"> + <country>Armenia</country> + </tld> + + <tld domain="an"> + <country>Netherlands Antilles</country> + </tld> + + <tld domain="ao"> + <country>Angola</country> + </tld> + + <tld domain="aq"> + <country>Antarctica</country> + </tld> + + <tld domain="ar"> + <country>Argentina</country> + </tld> + + <tld domain="as"> + <country>American Samoa</country> + </tld> + + <tld domain="at"> + <country>Austria</country> + </tld> + + <tld domain="au"> + <country>Australia</country> + </tld> + + <tld domain="aw"> + <country>Aruba</country> + </tld> + + <tld domain="ax"> + <country>Aland Islands</country> + </tld> + + <tld domain="az"> + <country>Azerbaijan</country> + </tld> + + <tld domain="ba"> + <country>Bosnia and Herzegovina</country> + </tld> + + <tld domain="bb"> + <country>Barbados</country> + </tld> + + <tld domain="bd"> + <country>Bangladesh</country> + </tld> + + <tld domain="be"> + <country>Belgium</country> + </tld> + + <tld domain="bf"> + <country>Burkina Faso</country> + </tld> + + <tld domain="bg"> + <country>Bulgaria</country> + </tld> + + <tld domain="bh"> + <country>Bahrain</country> + </tld> + + <tld domain="bi"> + <country>Burundi</country> + </tld> + + <tld domain="bj"> + <country>Benin</country> + </tld> + + <tld domain="bm"> + <country>Bermuda</country> + </tld> + + <tld domain="bn"> + <country>Brunei</country> + </tld> + + <tld domain="bo"> + <country>Bolivia</country> + </tld> + + <tld domain="br"> + <country>Brazil</country> + </tld> + + <tld domain="bs"> + <country>Bahamas</country> + </tld> + + <tld domain="bt"> + <country>Bhutan</country> + </tld> + + <tld domain="bu"> + <country>Burma</country> + <status>NOT_IN_USE</status> + <description> + not in use since re-naming of country to Myanmar, see .mm + </description> + </tld> + + <tld domain="bv"> + <country>Bouvet Island</country> + <status>NOT_IN_USE</status> + <description>not in use; no registrations</description> + </tld> + + <tld domain="bw"> + <country>Botswana</country> + </tld> + + <tld domain="by"> + <country>Belarus</country> + </tld> + + <tld domain="bz"> + <country>Belize</country> + </tld> + + <tld domain="ca"> + <country>Canada</country> + </tld> + + <tld domain="cc"> + <country>Cocos Keeling Islands</country> + </tld> + + <tld domain="cd"> + <country>Democratic Republic of the Congo</country> + <description>formerly .zr - Zaire</description> + </tld> + + <tld domain="cf"> + <country>Central African Republic</country> + </tld> + + <tld domain="cg"> + <country>Republic of the Congo</country> + </tld> + + <tld domain="ch"> + <country>Switzerland</country> + </tld> + + <tld domain="ci"> + <country>CÃŽte d'Ivoire</country> + <description>Ivory Coast</description> + </tld> + + <tld domain="ck"> + <country>Cook Islands</country> + </tld> + + <tld domain="cl"> + <country>Chile</country> + </tld> + + <tld domain="cm"> + <country>Cameroon</country> + </tld> + + <tld domain="cn"> + <country>People s Republic of China</country> + </tld> + + <tld domain="co"> + <country>Colombia</country> + </tld> + + <tld domain="cr"> + <country>Costa Rica</country> + </tld> + + <tld domain="cs"> + <country>Serbia and Montenegro</country> + <status>DELETED</status> + <description> + formerly .yu - Yugoslavia; description: on June 3, 2006, + Montenegro declared independence, thus dissolving the state + union) (.cs code not assigned; no DNS) (.cs code previously + used for Czechoslovakia + </description> + </tld> + + <tld domain="cu"> + <country>Cuba</country> + </tld> + + <tld domain="cv"> + <country>Cape Verde</country> + </tld> + + <tld domain="cx"> + <country>Christmas Island</country> + </tld> + + <tld domain="cy"> + <country>Cyprus</country> + </tld> + + <tld domain="cz"> + <country>Czech Republic</country> + </tld> + + <tld domain="dd"> + <country>German Democratic Republic(East Germany)</country> + <status>DELETED</status> + <description>deleted in 1990</description> + </tld> + + <tld domain="de"> + <country>Germany</country> + </tld> + + <tld domain="dj"> + <country>Djibouti</country> + </tld> + + <tld domain="dk"> + <country>Denmark</country> + </tld> + + <tld domain="dm"> + <country>Dominica</country> + </tld> + + <tld domain="do"> + <country>Dominican Republic</country> + </tld> + + <tld domain="dz"> + <country>Algeria</country> + </tld> + + <tld domain="ec"> + <country>Ecuador</country> + </tld> + + <tld domain="ee"> + <country>Estonia</country> + </tld> + + <tld domain="eg"> + <country>Egypt</country> + </tld> + + <tld domain="eh"> + <country>Western Sahara</country> + <status>NOT_IN_USE</status> + <description>not assigned; no DNS</description> + </tld> + + <tld domain="er"> + <country>Eritrea</country> + </tld> + + <tld domain="es"> + <country>Spain</country> + </tld> + + <tld domain="et"> + <country>Ethiopia</country> + </tld> + + <tld domain="eu"> + <country>European Union</country> + <description> + code "exceptionally reserved" by ISO 3166-1 + </description> + </tld> + + <tld domain="fi"> + <country>Finland</country> + </tld> + + <tld domain="fj"> + <country>Fiji</country> + </tld> + + <tld domain="fk"> + <country>Falkland Islands</country> + </tld> + + <tld domain="fm"> + <country>Federated States of Micronesia</country> + </tld> + + <tld domain="fo"> + <country>Faroe Islands</country> + </tld> + + <tld domain="fr"> + <country>France</country> + </tld> + + <tld domain="ga"> + <country>Gabon</country> + </tld> + + <tld domain="gb"> + <country>United Kingdom</country> + <description> + Reserved domain by IANA; deprecated – see .uk + </description> + </tld> + + <tld domain="gd"> + <country>Grenada</country> + </tld> + + <tld domain="ge"> + <country>Georgia</country> + </tld> + + <tld domain="gf"> + <country>French Guiana</country> + </tld> + + <tld domain="gg"> + <country>Guernsey</country> + </tld> + + <tld domain="gh"> + <country>Ghana</country> + </tld> + + <tld domain="gi"> + <country>Gibraltar</country> + </tld> + + <tld domain="gl"> + <country>Greenland</country> + </tld> + + <tld domain="gm"> + <country>Gambia</country> + </tld> + + <tld domain="gn"> + <country>Guinea</country> + </tld> + + <tld domain="gp"> + <country>Guadeloupe</country> + </tld> + + <tld domain="gq"> + <country>Equatorial Guinea</country> + </tld> + + <tld domain="gr"> + <country>Greece</country> + </tld> + + <tld domain="gs"> + <country>South Georgia and the South Sandwich Islands</country> + </tld> + + <tld domain="gt"> + <country>Guatemala</country> + </tld> + + <tld domain="gu"> + <country>Guam</country> + </tld> + + <tld domain="gw"> + <country>Guinea Bissau</country> + </tld> + + <tld domain="gy"> + <country>Guyana</country> + </tld> + + <tld domain="hk"> + <country>Hong Kong</country> + </tld> + + <tld domain="hm"> + <country>Heard Island and McDonald Islands</country> + </tld> + + <tld domain="hn"> + <country>Honduras</country> + </tld> + + <tld domain="hr"> + <country>Croatia</country> + </tld> + + <tld domain="ht"> + <country>Haiti</country> + </tld> + + <tld domain="hu"> + <country>Hungary</country> + </tld> + + <tld domain="id"> + <country>Indonesia</country> + </tld> + + <tld domain="ie"> + <country>Ireland</country> + </tld> + + <tld domain="il"> + <country>Israel</country> + </tld> + + <tld domain="im"> + <country>Isle of Man</country> + </tld> + + <tld domain="in"> + <country>India</country> + </tld> + + <tld domain="io"> + <country>British Indian Ocean Territory</country> + </tld> + + <tld domain="iq"> + <country>Iraq</country> + </tld> + + <tld domain="ir"> + <country>Iran</country> + </tld> + + <tld domain="is"> + <country>Iceland</country> + </tld> + + <tld domain="it"> + <country>Italy</country> + </tld> + + <tld domain="je"> + <country>Jersey</country> + </tld> + + <tld domain="jm"> + <country>Jamaica</country> + </tld> + + <tld domain="jo"> + <country>Jordan</country> + </tld> + + <tld domain="jp"> + <country>Japan</country> + </tld> + + <tld domain="ke"> + <country>Kenya</country> + </tld> + + <tld domain="kg"> + <country>Kyrgyzstan</country> + </tld> + + <tld domain="kh"> + <country>Cambodia</country> + </tld> + + <tld domain="ki"> + <country>Kiribati</country> + </tld> + + <tld domain="km"> + <country>Comoros</country> + </tld> + + <tld domain="kn"> + <country>Saint Kitts and Nevis</country> + </tld> + + <tld domain="kp"> + <country>North Korea</country> + <status>NOT_IN_USE</status> + + <description>not assigned; no DNS</description> + </tld> + + <tld domain="kr"> + <country>South Korea</country> + </tld> + + <tld domain="kw"> + <country>Kuwait</country> + </tld> + + <tld domain="ky"> + <country>Cayman Islands</country> + </tld> + + <tld domain="kz"> + <country>Kazakhstan</country> + </tld> + + <tld domain="la"> + <country>Laos</country> + </tld> + + <tld domain="lb"> + <country>Lebanon</country> + </tld> + + <tld domain="lc"> + <country>Saint Lucia</country> + </tld> + + <tld domain="li"> + <country>Liechtenstein</country> + </tld> + + <tld domain="lk"> + <country>Sri Lanka</country> + </tld> + + <tld domain="lr"> + <country>Liberia</country> + </tld> + + <tld domain="ls"> + <country>Lesotho</country> + </tld> + + <tld domain="lt"> + <country>Lithuania</country> + </tld> + + <tld domain="lu"> + <country>Luxembourg</country> + </tld> + + <tld domain="lv"> + <country>Latvia</country> + </tld> + + <tld domain="ly"> + <country>Libya</country> + </tld> + + <tld domain="ma"> + <country>Morocco</country> + </tld> + + <tld domain="mc"> + <country>Monaco</country> + </tld> + + <tld domain="md"> + <country>Moldova</country> + </tld> + + <tld domain="me"> + <country>Montenegro</country> + </tld> + + <tld domain="mg"> + <country>Madagascar</country> + </tld> + + <tld domain="mh"> + <country>Marshall Islands</country> + </tld> + + <tld domain="mk"> + <country>Republic of Macedonia</country> + </tld> + + <tld domain="ml"> + <country>Mali</country> + </tld> + + <tld domain="mm"> + <country>Myanmar</country> + <description>formerly .bu - Burma</description> + </tld> + + <tld domain="mn"> + <country>Mongolia</country> + </tld> + + <tld domain="mo"> + <country>Macau</country> + </tld> + + <tld domain="mp"> + <country>Northern Mariana Islands</country> + </tld> + + <tld domain="mq"> + <country>Martinique</country> + </tld> + + <tld domain="mr"> + <country>Mauritania</country> + </tld> + + <tld domain="ms"> + <country>Montserrat</country> + </tld> + + <tld domain="mt"> + <country>Malta</country> + </tld> + + <tld domain="mu"> + <country>Mauritius</country> + </tld> + + <tld domain="mv"> + <country>Maldives</country> + </tld> + + <tld domain="mw"> + <country>Malawi</country> + </tld> + + <tld domain="mx"> + <country>Mexico</country> + </tld> + + <tld domain="my"> + <country>Malaysia</country> + </tld> + + <tld domain="mz"> + <country>Mozambique</country> + </tld> + + <tld domain="na"> + <country>Namibia</country> + </tld> + + <tld domain="nc"> + <country>New Caledonia</country> + </tld> + + <tld domain="ne"> + <country>Niger</country> + </tld> + + <tld domain="nf"> + <country>Norfolk Island</country> + </tld> + + <tld domain="ng"> + <country>Nigeria</country> + </tld> + + <tld domain="ni"> + <country>Nicaragua</country> + </tld> + + <tld domain="nl"> + <country>Netherlands</country> + </tld> + + <tld domain="no"> + <country>Norway</country> + </tld> + + <tld domain="np"> + <country>Nepal</country> + </tld> + + <tld domain="nr"> + <country>Nauru</country> + </tld> + + <tld domain="nu"> + <country>Niue</country> + </tld> + + <tld domain="nz"> + <country>New Zealand</country> + </tld> + + <tld domain="om"> + <country>Oman</country> + </tld> + + <tld domain="pa"> + <country>Panama</country> + </tld> + + <tld domain="pe"> + <country>Peru</country> + </tld> + + <tld domain="pf"> + <country>French Polynesia</country> + </tld> + + <tld domain="pg"> + <country>Papua New Guinea</country> + </tld> + + <tld domain="ph"> + <country>Philippines</country> + </tld> + + <tld domain="pk"> + <country>Pakistan</country> + </tld> + + <tld domain="pl"> + <country>Poland</country> + </tld> + + <tld domain="pm"> + <country>Saint Pierre and Miquelon</country> + </tld> + + <tld domain="pn"> + <country>Pitcairn Islands</country> + </tld> + + <tld domain="pr"> + <country>Puerto Rico</country> + </tld> + + <tld domain="ps"> + <country>Palestinian territories</country> + </tld> + + <tld domain="pt"> + <country>Portugal</country> + </tld> + + <tld domain="pw"> + <country>Palau</country> + </tld> + + <tld domain="py"> + <country>Paraguay</country> + </tld> + + <tld domain="qa"> + <country>Qatar</country> + </tld> + + <tld domain="re"> + <country>Réunion</country> + </tld> + + <tld domain="ro"> + <country>Romania</country> + </tld> + + <tld domain="rs"> + <country>Serbia</country> + </tld> + + <tld domain="ru"> + <country>Russia</country> + </tld> + + <tld domain="rw"> + <country>Rwanda</country> + </tld> + + <tld domain="sa"> + <country>Saudi Arabia</country> + </tld> + + <tld domain="sb"> + <country>Solomon Islands</country> + </tld> + + <tld domain="sc"> + <country>Seychelles</country> + </tld> + + <tld domain="sd"> + <country>Sudan</country> + </tld> + + <tld domain="se"> + <country>Sweden</country> + </tld> + + <tld domain="sg"> + <country>Singapore</country> + </tld> + + <tld domain="sh"> + <country>Saint Helena</country> + </tld> + + <tld domain="si"> + <country>Slovenia</country> + </tld> + + <tld domain="sj"> + <country>Svalbard and Jan Mayen Islands</country> + <status>NOT_IN_USE</status> + <description>not in use; no registrations</description> + </tld> + + <tld domain="sk"> + <country>Slovakia</country> + </tld> + + <tld domain="sl"> + <country>Sierra Leone</country> + </tld> + + <tld domain="sm"> + <country>San Marino</country> + </tld> + + <tld domain="sn"> + <country>Senegal</country> + </tld> + + <tld domain="so"> + <country>Somalia</country> + </tld> + + <tld domain="sr"> + <country>Suriname</country> + </tld> + + <tld domain="st"> + <country>São Tomé and PrÃÂncipe</country> + </tld> + + <tld domain="su"> + <country>Soviet Union</country> + <status>DELETED</status> + <description> + deprecated; being phased out; code "transitionally reserved" + by ISO 3166-1 + </description> + </tld> + + <tld domain="sv"> + <country>El Salvador</country> + </tld> + + <tld domain="sy"> + <country>Syria</country> + </tld> + + <tld domain="sz"> + <country>Swaziland</country> + </tld> + + <tld domain="tc"> + <country>Turks and Caicos Islands</country> + </tld> + + <tld domain="td"> + <country>Chad</country> + </tld> + + <tld domain="tf"> + <country>French Southern Territories</country> + </tld> + + <tld domain="tg"> + <country>Togo</country> + </tld> + + <tld domain="th"> + <country>Thailand</country> + </tld> + + <tld domain="tj"> + <country>Tajikistan</country> + </tld> + + <tld domain="tk"> + <country>Tokelau</country> + </tld> + + <tld domain="tl"> + <country>East Timor</country> + <description>formerly .tp</description> + </tld> + + <tld domain="tm"> + <country>Turkmenistan</country> + </tld> + + <tld domain="tn"> + <country>Tunisia</country> + </tld> + + <tld domain="to"> + <country>Tonga</country> + </tld> + + <tld domain="tp"> + <country>East Timor</country> + <status>DELETED</status> + <description> + deprecated - use .tl; code "transitionally reserved" by ISO + 3166-1 + </description> + </tld> + + <tld domain="tr"> + <country>Turkey</country> + </tld> + + <tld domain="tt"> + <country>Trinidad and Tobago</country> + </tld> + + <tld domain="tv"> + <country>Tuvalu</country> + </tld> + + <tld domain="tw"> + <country>Republic of China</country> + <description>Taiwan</description> + </tld> + + <tld domain="tz"> + <country>Tanzania</country> + </tld> + + <tld domain="ua"> + <country>Ukraine</country> + </tld> + + <tld domain="ug"> + <country>Uganda</country> + </tld> + + <tld domain="uk"> + <country>United Kingdom</country> + <description> + code "exceptionally reserved" by ISO 3166-1 (see also .gb) + </description> + </tld> + + <tld domain="um"> + <country>United States Minor Outlying Islands</country> + <status>DELETED</status> + <description>see http://en.wikipedia.org/wiki/.um</description> + </tld> + + <tld domain="us"> + <country>United States</country> + </tld> + + <tld domain="uy"> + <country>Uruguay</country> + </tld> + + <tld domain="uz"> + <country>Uzbekistan</country> + </tld> + + <tld domain="va"> + <country>Vatican City</country> + </tld> + + <tld domain="vc"> + <country>Saint Vincent and the Grenadines</country> + </tld> + + <tld domain="ve"> + <country>Venezuela</country> + </tld> + + <tld domain="vg"> + <country>British Virgin Islands</country> + </tld> + + <tld domain="vi"> + <country>United States Virgin Islands</country> + </tld> + + <tld domain="vn"> + <country>Vietnam</country> + </tld> + + <tld domain="vu"> + <country>Vanuatu</country> + </tld> + + <tld domain="wf"> + <country>Wallis and Futuna</country> + </tld> + + <tld domain="ws"> + <country>Samoa</country> + <description>formerly Western Samoa</description> + </tld> + + <tld domain="ye"> + <country>Yemen</country> + </tld> + + <tld domain="yt"> + <country>Mayotte</country> + </tld> + + <tld domain="yu"> + <country>Yugoslavia</country> + <description> + subsequently renamed Serbia and Montenegro (code officially + replaced by .cs (see above) but still used; code + "transitionally reserved" by ISO 3166-1) + </description> + </tld> + + <tld domain="za"> + <country>South Africa</country> + </tld> + + <tld domain="zm"> + <country>Zambia</country> + </tld> + + <tld domain="zr"> + <country>Zaire</country> + <status>DELETED</status> + <description>replaced by .cd</description> + </tld> + + <tld domain="zw"> + <country>Zimbabwe</country> + </tld> + </cctlds> + </tlds> + + <!-- Second Level Domains --> + <suffixes> + + <!-- .us Second Level Domains --> + <suffix domain="as.us" /> + <suffix domain="gu.us" /> + <suffix domain="pr.us" /> + <suffix domain="vi.us" /> + + <!-- .uk Second Level Domains --> + <suffix domain="ac.uk" /> + <suffix domain="co.uk" /> + <suffix domain="gov.uk" /> + <suffix domain="ltd.uk" /> + <suffix domain="me.uk" /> + <suffix domain="mod.uk" /> + <suffix domain="net.uk" /> + <suffix domain="nic.uk" /> + <suffix domain="nhs.uk" /> + <suffix domain="org.uk" /> + <suffix domain="pic.uk" /> + <suffix domain="police.uk" /> + <suffix domain="govt.uk"> + <status>DELETED</status> + </suffix> + <suffix domain="orgn.uk"> + <status>DELETED</status> + </suffix> + <suffix domain="mil.uk"> + <status>DELETED</status> + </suffix> + <suffix domain="lea.uk"> + <status>DELETED</status> + </suffix> + + + <!-- .tr Second Level Domains --> + <suffix domain="com.tr" /> + <suffix domain="gen.tr" /> + <suffix domain="org.tr"/> + <suffix domain="biz.tr" /> + <suffix domain="info.tr" /> + <suffix domain="av.tr" /> + <suffix domain="dr.tr" /> + <suffix domain="pol.tr" /> + <suffix domain="bel.tr" /> + <suffix domain="mil.tr" /> + <suffix domain="mil.tr" /> + <suffix domain="bbs.tr" /> + <suffix domain="k12.tr" /> + <suffix domain="edu.tr"/> + <suffix domain="name.tr" /> + <suffix domain="net.tr" /> + <suffix domain="gov.tr" /> + <suffix domain="gov.tr" /> + <suffix domain="web.tr" /> + <suffix domain="tel.tr" /> + <suffix domain="tv.tr" /> + + <!-- .hk Second Level Domains --> + <suffix domain="ust.hk" /> + <suffix domain="hku.hk" /> + <suffix domain="cuhk.hk" /> + + <!-- .it Second Level Domains --> + + <suffix domain="edu.it" /> + <suffix domain="gov.it" /> + <suffix domain="italy.it" /> + + <!-- .az Second Level Domains --> + <suffix domain="pro.az" /> + + <!-- .au Second Level Domains --> + + <suffix domain="com.au" /> + <suffix domain="net.au" /> + <suffix domain="gov.au" /> + <suffix domain="org.au" /> + <suffix domain="edu.au" /> + <suffix domain="csiro.au" /> + <suffix domain="asn.au" /> + <suffix domain="id.au" /> + <!-- New second level domains --> + <suffix domain="act.au" /> + <suffix domain="nsw.au" /> + <suffix domain="nt.au" /> + <suffix domain="qld.au" /> + <suffix domain="sa.au" /> + <suffix domain="tas.au" /> + <suffix domain="vic.au" /> + <suffix domain="wa.au" /> + + <!-- .int Second Level Domains --> + <suffix domain="eu.int" /> + + <!-- .br Second Level Domains --> + <suffix domain="adm.br" /> + <suffix domain="adv.br" /> + <suffix domain="agr.br" /> + <suffix domain="am.br" /> + <suffix domain="arq.br" /> + <suffix domain="art.br" /> + <suffix domain="ato.br" /> + <suffix domain="bio.br" /> + <suffix domain="blog.br" /> + <suffix domain="bmd.br" /> + <suffix domain="cim.br" /> + <suffix domain="cng.br" /> + <suffix domain="cnt.br" /> + <suffix domain="com.br" /> + <suffix domain="coop.br" /> + <suffix domain="ecn.br" /> + <suffix domain="edu.br" /> + <suffix domain="eng.br" /> + <suffix domain="esp.br" /> + <suffix domain="etc.br" /> + <suffix domain="eti.br" /> + <suffix domain="far.br" /> + <suffix domain="flog.br" /> + <suffix domain="fm.br" /> + <suffix domain="fnd.br" /> + <suffix domain="fot.br" /> + <suffix domain="fst.br" /> + <suffix domain="g12.br" /> + <suffix domain="ggf.br" /> + <suffix domain="gov.br" /> + <suffix domain="imb.br" /> + <suffix domain="ind.br" /> + <suffix domain="inf.br" /> + <suffix domain="jor.br" /> + <suffix domain="lel.br" /> + <suffix domain="mat.br" /> + <suffix domain="med.br" /> + <suffix domain="mil.br" /> + <suffix domain="mus.br" /> + <suffix domain="net.br" /> + <suffix domain="nom.br" /> + <suffix domain="not.br" /> + <suffix domain="ntr.br" /> + <suffix domain="odo.br" /> + <suffix domain="org.br" /> + <suffix domain="ppg.br" /> + <suffix domain="pro.br" /> + <suffix domain="psc.br" /> + <suffix domain="psi.br" /> + <suffix domain="qsl.br" /> + <suffix domain="rec.br" /> + <suffix domain="slg.br" /> + <suffix domain="srv.br" /> + <suffix domain="tmp.br" /> + <suffix domain="trd.br" /> + <suffix domain="tur.br" /> + <suffix domain="tv.br" /> + <suffix domain="vet.br" /> + <suffix domain="vlog.br" /> + <suffix domain="wiki.br" /> + <suffix domain="zlg.br" /> + + <!-- + Below elements are generated from the file + at http://publicsuffix.org/. + see org.apache.nutch.util.domain.MozillaPublicSuffixListParser.java + --> + + <!-- ac : http://en.wikipedia.org/wiki/.am--> + <suffix domain="com.ac" /> + <suffix domain="edu.ac" /> + <suffix domain="gov.ac" /> + <suffix domain="net.ac" /> + <suffix domain="mil.ac" /> + <suffix domain="org.ac" /> + + <!-- ad : http://en.wikipedia.org/wiki/.ad--> + <suffix domain="nom.ad" /> + + <!-- ae : http://en.wikipedia.org/wiki/.ae--> + <suffix domain="net.ae" /> + <suffix domain="gov.ae" /> + <suffix domain="ac.ae" /> + <suffix domain="sch.ae" /> + <suffix domain="org.ae" /> + <suffix domain="mil.ae" /> + <suffix domain="pro.ae" /> + <suffix domain="name.ae" /> + + <!-- aero : see http://www.information.aero/index.php?id=66--> + <suffix domain="accident-investigation.aero" /> + <suffix domain="accident-prevention.aero" /> + <suffix domain="aerobatic.aero" /> + <suffix domain="aeroclub.aero" /> + <suffix domain="aerodrome.aero" /> + <suffix domain="agents.aero" /> + <suffix domain="aircraft.aero" /> + <suffix domain="airline.aero" /> + <suffix domain="airport.aero" /> + <suffix domain="air-surveillance.aero" /> + <suffix domain="airtraffic.aero" /> + <suffix domain="air-traffic-control.aero" /> + <suffix domain="ambulance.aero" /> + <suffix domain="amusement.aero" /> + <suffix domain="association.aero" /> + <suffix domain="author.aero" /> + <suffix domain="ballooning.aero" /> + <suffix domain="broker.aero" /> + <suffix domain="caa.aero" /> + <suffix domain="cargo.aero" /> + <suffix domain="catering.aero" /> + <suffix domain="certification.aero" /> + <suffix domain="championship.aero" /> + <suffix domain="charter.aero" /> + <suffix domain="civilaviation.aero" /> + <suffix domain="club.aero" /> + <suffix domain="conference.aero" /> + <suffix domain="consultant.aero" /> + <suffix domain="consulting.aero" /> + <suffix domain="control.aero" /> + <suffix domain="council.aero" /> + <suffix domain="crew.aero" /> + <suffix domain="design.aero" /> + <suffix domain="dgca.aero" /> + <suffix domain="educator.aero" /> + <suffix domain="emergency.aero" /> + <suffix domain="engine.aero" /> + <suffix domain="engineer.aero" /> + <suffix domain="entertainment.aero" /> + <suffix domain="equipment.aero" /> + <suffix domain="exchange.aero" /> + <suffix domain="express.aero" /> + <suffix domain="federation.aero" /> + <suffix domain="flight.aero" /> + <suffix domain="freight.aero" /> + <suffix domain="fuel.aero" /> + <suffix domain="gliding.aero" /> + <suffix domain="government.aero" /> + <suffix domain="groundhandling.aero" /> + <suffix domain="group.aero" /> + <suffix domain="hanggliding.aero" /> + <suffix domain="homebuilt.aero" /> + <suffix domain="insurance.aero" /> + <suffix domain="journal.aero" /> + <suffix domain="journalist.aero" /> + <suffix domain="leasing.aero" /> + <suffix domain="logistics.aero" /> + <suffix domain="magazine.aero" /> + <suffix domain="maintenance.aero" /> + <suffix domain="marketplace.aero" /> + <suffix domain="media.aero" /> + <suffix domain="microlight.aero" /> + <suffix domain="modelling.aero" /> + <suffix domain="navigation.aero" /> + <suffix domain="parachuting.aero" /> + <suffix domain="paragliding.aero" /> + <suffix domain="passenger-association.aero" /> + <suffix domain="pilot.aero" /> + <suffix domain="press.aero" /> + <suffix domain="production.aero" /> + <suffix domain="recreation.aero" /> + <suffix domain="repbody.aero" /> + <suffix domain="res.aero" /> + <suffix domain="research.aero" /> + <suffix domain="rotorcraft.aero" /> + <suffix domain="safety.aero" /> + <suffix domain="scientist.aero" /> + <suffix domain="services.aero" /> + <suffix domain="show.aero" /> + <suffix domain="skydiving.aero" /> + <suffix domain="software.aero" /> + <suffix domain="student.aero" /> + <suffix domain="taxi.aero" /> + <suffix domain="trader.aero" /> + <suffix domain="trading.aero" /> + <suffix domain="trainer.aero" /> + <suffix domain="union.aero" /> + <suffix domain="workinggroup.aero" /> + <suffix domain="works.aero" /> + + <!-- af : http://www.nic.af/help.jsp--> + <suffix domain="gov.af" /> + <suffix domain="com.af" /> + <suffix domain="org.af" /> + <suffix domain="net.af" /> + <suffix domain="edu.af" /> + + <!-- ag : http://www.nic.ag/prices.htm--> + <suffix domain="com.ag" /> + <suffix domain="org.ag" /> + <suffix domain="net.ag" /> + <suffix domain="co.ag" /> + <suffix domain="nom.ag" /> + + <!-- ai : http://nic.com.ai/--> + <suffix domain="off.ai" /> + <suffix domain="com.ai" /> + <suffix domain="net.ai" /> + <suffix domain="org.ai" /> + + <!-- al : http://www.inima.al/Domains.html--> + <suffix domain="gov.al" /> + <suffix domain="edu.al" /> + <suffix domain="org.al" /> + <suffix domain="com.al" /> + <suffix domain="net.al" /> + + <!-- am : http://en.wikipedia.org/wiki/.am--> + + <!-- an : http://www.una.an/an_domreg/default.asp--> + <suffix domain="com.an" /> + <suffix domain="net.an" /> + <suffix domain="org.an" /> + <suffix domain="edu.an" /> + + <!-- ao : http://en.wikipedia.org/wiki/.ao--> + <!-- list of 2nd level TLDs ?--> + + <!-- aq : http://en.wikipedia.org/wiki/.aq--> + + <!-- ar : http://en.wikipedia.org/wiki/.ar--> + <suffix domain="congresodelalengua3.ar" /> + <suffix domain="educ.ar" /> + <suffix domain="gobiernoelectronico.ar" /> + <suffix domain="mecon.ar" /> + <suffix domain="nacion.ar" /> + <suffix domain="nic.ar" /> + <suffix domain="promocion.ar" /> + <suffix domain="retina.ar" /> + <suffix domain="uba.ar" /> + + <!-- arpa : http://en.wikipedia.org/wiki/.arpa--> + <suffix domain="e164.arpa" /> + <suffix domain="in-addr.arpa" /> + <suffix domain="ip6.arpa" /> + <suffix domain="uri.arpa" /> + <suffix domain="urn.arpa" /> + + <!-- as : http://en.wikipedia.org/wiki/.as--> + + <!-- at : http://en.wikipedia.org/wiki/.at--> + <suffix domain="gv.at" /> + <suffix domain="ac.at" /> + <suffix domain="co.at" /> + <suffix domain="or.at" /> + + <!-- au : http://en.wikipedia.org/wiki/.au--> + <!-- au geographical names (vic.au etc... are covered above)--> + <suffix domain="act.edu.au" /> + <suffix domain="nsw.edu.au" /> + <suffix domain="nt.edu.au" /> + <suffix domain="qld.edu.au" /> + <suffix domain="sa.edu.au" /> + <suffix domain="tas.edu.au" /> + <suffix domain="vic.edu.au" /> + <suffix domain="wa.edu.au" /> + <suffix domain="act.gov.au" /> + <suffix domain="nsw.gov.au" /> + <suffix domain="nt.gov.au" /> + <suffix domain="qld.gov.au" /> + <suffix domain="sa.gov.au" /> + <suffix domain="tas.gov.au" /> + <suffix domain="vic.gov.au" /> + <suffix domain="wa.gov.au" /> + + <!-- aw : http://en.wikipedia.org/wiki/.aw--> + <suffix domain="com.aw" /> + + <!-- ax : http://en.wikipedia.org/wiki/.ax--> + + <!-- az : http://en.wikipedia.org/wiki/.az--> + <suffix domain="com.az" /> + <suffix domain="net.az" /> + <suffix domain="int.az" /> + <suffix domain="gov.az" /> + <suffix domain="org.az" /> + <suffix domain="edu.az" /> + <suffix domain="info.az" /> + <suffix domain="pp.az" /> + <suffix domain="mil.az" /> + <suffix domain="name.az" /> + <suffix domain="biz.az" /> + + <!-- ba : http://en.wikipedia.org/wiki/.ba--> + <suffix domain="org.ba" /> + <suffix domain="net.ba" /> + <suffix domain="edu.ba" /> + <suffix domain="gov.ba" /> + <suffix domain="mil.ba" /> + <suffix domain="unsa.ba" /> + <suffix domain="unbi.ba" /> + <suffix domain="co.ba" /> + <suffix domain="com.ba" /> + <suffix domain="rs.ba" /> + + <!-- bb : http://en.wikipedia.org/wiki/.bb--> + <suffix domain="com.bb" /> + <suffix domain="edu.bb" /> + <suffix domain="gov.bb" /> + <suffix domain="net.bb" /> + <suffix domain="org.bb" /> + + <!-- bd : http://en.wikipedia.org/wiki/.bd--> + + <!-- be : http://en.wikipedia.org/wiki/.be--> + <suffix domain="ac.be" /> + + <!-- bf : http://en.wikipedia.org/wiki/.bf--> + + <!-- bg : http://en.wikipedia.org/wiki/.bg--> + + <!-- bh : http://en.wikipedia.org/wiki/.bh--> + <!-- list of 2nd level tlds ?--> + + <!-- bi : http://en.wikipedia.org/wiki/.bi--> + <!-- list of 2nd level tlds ?--> + + <!-- biz : http://en.wikipedia.org/wiki/.biz--> + + <!-- bj : http://en.wikipedia.org/wiki/.bj--> + <!-- list of 2nd level tlds ?--> + + <!-- bm : http://www.bermudanic.bm/dnr-text.txt--> + <suffix domain="com.bm" /> + <suffix domain="edu.bm" /> + <suffix domain="gov.bm" /> + <suffix domain="net.bm" /> + <suffix domain="org.bm" /> + + <!-- bn : http://en.wikipedia.org/wiki/.bn--> + + <!-- bo : http://www.nic.bo/--> + <suffix domain="com.bo" /> + <suffix domain="edu.bo" /> + <suffix domain="gov.bo" /> + <suffix domain="gob.bo" /> + <suffix domain="int.bo" /> + <suffix domain="org.bo" /> + <suffix domain="net.bo" /> + <suffix domain="mil.bo" /> + <suffix domain="tv.bo" /> + + <!-- br : http://en.wikipedia.org/wiki/.br--> + + <!-- bs : http://www.nic.bs/rules.html--> + <suffix domain="com.bs" /> + <suffix domain="net.bs" /> + <suffix domain="org.bs" /> + <suffix domain="edu.bs" /> + <suffix domain="gov.bs" /> + + <!-- bt : http://en.wikipedia.org/wiki/.bt--> + + <!-- bw : http://en.wikipedia.org/wiki/.bw--> + <!-- list of 2nd level tlds ?--> + + <!-- by : http://en.wikipedia.org/wiki/.by--> + <!-- list of 2nd level tlds ?--> + + <!-- bz : http://en.wikipedia.org/wiki/.bz--> + <!-- list of 2nd level tlds ?--> + + <!-- ca : http://en.wikipedia.org/wiki/.ca--> + <!-- ca geographical names--> + <suffix domain="ab.ca" /> + <suffix domain="bc.ca" /> + <suffix domain="mb.ca" /> + <suffix domain="nb.ca" /> + <suffix domain="nf.ca" /> + <suffix domain="nl.ca" /> + <suffix domain="ns.ca" /> + <suffix domain="nt.ca" /> + <suffix domain="nu.ca" /> + <suffix domain="on.ca" /> + <suffix domain="pe.ca" /> + <suffix domain="qc.ca" /> + <suffix domain="sk.ca" /> + <suffix domain="yk.ca" /> + + <!-- cat : http://en.wikipedia.org/wiki/.cat--> + + <!-- cc : http://en.wikipedia.org/wiki/.cc--> + + <!-- cd : http://en.wikipedia.org/wiki/.cd--> + + <!-- cf : http://en.wikipedia.org/wiki/.cf--> + + <!-- cg : http://en.wikipedia.org/wiki/.cg--> + + <!-- ch : http://en.wikipedia.org/wiki/.ch--> + + <!-- ci : http://en.wikipedia.org/wiki/.ci--> + <!-- list of 2nd level tlds ?--> + + <!-- ck : http://en.wikipedia.org/wiki/.ck--> + + <!-- cl : http://en.wikipedia.org/wiki/.cl--> + + <!-- cm : http://en.wikipedia.org/wiki/.cm--> + + <!-- cn : http://en.wikipedia.org/wiki/.cn--> + <suffix domain="ac.cn" /> + <suffix domain="com.cn" /> + <suffix domain="edu.cn" /> + <suffix domain="gov.cn" /> + <suffix domain="net.cn" /> + <suffix domain="org.cn" /> + <!-- cn geographic names--> + <suffix domain="ah.cn" /> + <suffix domain="bj.cn" /> + <suffix domain="cq.cn" /> + <suffix domain="fj.cn" /> + <suffix domain="gd.cn" /> + <suffix domain="gs.cn" /> + <suffix domain="gz.cn" /> + <suffix domain="gx.cn" /> + <suffix domain="ha.cn" /> + <suffix domain="hb.cn" /> + <suffix domain="he.cn" /> + <suffix domain="hi.cn" /> + <suffix domain="hl.cn" /> + <suffix domain="hn.cn" /> + <suffix domain="jl.cn" /> + <suffix domain="js.cn" /> + <suffix domain="jx.cn" /> + <suffix domain="ln.cn" /> + <suffix domain="nm.cn" /> + <suffix domain="nx.cn" /> + <suffix domain="qh.cn" /> + <suffix domain="sc.cn" /> + <suffix domain="sd.cn" /> + <suffix domain="sh.cn" /> + <suffix domain="sn.cn" /> + <suffix domain="sx.cn" /> + <suffix domain="tj.cn" /> + <suffix domain="xj.cn" /> + <suffix domain="xz.cn" /> + <suffix domain="yn.cn" /> + <suffix domain="zj.cn" /> + + <!-- co : http://en.wikipedia.org/wiki/.co--> + + <!-- com : http://en.wikipedia.org/wiki/.com--> + + <!-- coop : http://en.wikipedia.org/wiki/.coop--> + + <!-- cr : http://en.wikipedia.org/wiki/.cr--> + + <!-- cu : http://en.wikipedia.org/wiki/.cu--> + <suffix domain="com.cu" /> + <suffix domain="edu.cu" /> + <suffix domain="org.cu" /> + <suffix domain="net.cu" /> + <suffix domain="gov.cu" /> + <suffix domain="inf.cu" /> + + <!-- cv : http://en.wikipedia.org/wiki/.cv--> + + <!-- cx : http://en.wikipedia.org/wiki/.cx--> + + <!-- cy : http://en.wikipedia.org/wiki/.cy--> + + <!-- cz : http://en.wikipedia.org/wiki/.cz--> + + <!-- de : http://en.wikipedia.org/wiki/.de--> + + <!-- dj : http://en.wikipedia.org/wiki/.dj--> + + <!-- dk : http://en.wikipedia.org/wiki/.dk--> + + <!-- dm : http://en.wikipedia.org/wiki/.dm--> + <suffix domain="com.dm" /> + <suffix domain="net.dm" /> + <suffix domain="org.dm" /> + + <!-- do : http://en.wikipedia.org/wiki/.do--> + + <!-- dz : http://en.wikipedia.org/wiki/.dz--> + <suffix domain="com.dz" /> + <suffix domain="org.dz" /> + <suffix domain="net.dz" /> + <suffix domain="gov.dz" /> + <suffix domain="edu.dz" /> + <suffix domain="asso.dz" /> + <suffix domain="pol.dz" /> + <suffix domain="art.dz" /> + + <!-- ec : http://www.nic.ec/reg/paso1.asp--> + <suffix domain="com.ec" /> + <suffix domain="info.ec" /> + <suffix domain="net.ec" /> + <suffix domain="fin.ec" /> + <suffix domain="med.ec" /> + <suffix domain="pro.ec" /> + <suffix domain="org.ec" /> + <suffix domain="edu.ec" /> + <suffix domain="gov.ec" /> + <suffix domain="mil.ec" /> + + <!-- edu : http://en.wikipedia.org/wiki/.edu--> + + <!-- ee : http://www3.eenet.ee/ee/application.html--> + <suffix domain="com.ee" /> + <suffix domain="org.ee" /> + <suffix domain="fie.ee" /> + <suffix domain="pri.ee" /> + + <!-- eg : http://en.wikipedia.org/wiki/.eg--> + + <!-- er : http://en.wikipedia.org/wiki/.er--> + + <!-- es : https://www.nic.es/site_ingles/ingles/dominios/index.html--> + <suffix domain="com.es" /> + <suffix domain="nom.es" /> + <suffix domain="org.es" /> + <suffix domain="gob.es" /> + <suffix domain="edu.es" /> + + <!-- et : http://en.wikipedia.org/wiki/.et--> + + <!-- eu : http://en.wikipedia.org/wiki/.eu--> + + <!-- fi : http://en.wikipedia.org/wiki/.fi--> + + <!-- fj : http://en.wikipedia.org/wiki/.fj--> + + <!-- fk : http://en.wikipedia.org/wiki/.fk--> + + <!-- fm : http://en.wikipedia.org/wiki/.fm--> + + <!-- fo : http://en.wikipedia.org/wiki/.fo--> + + <!-- fr : http://www.afnic.fr/--> + <!-- domaines descriptifs : http://www.afnic.fr/obtenir/chartes/nommage-fr/annexe-descriptifs--> + <suffix domain="com.fr" /> + <suffix domain="asso.fr" /> + <suffix domain="nom.fr" /> + <suffix domain="prd.fr" /> + <suffix domain="presse.fr" /> + <suffix domain="tm.fr" /> + <!-- domaines sectoriels : http://www.afnic.fr/obtenir/chartes/nommage-fr/annexe-sectoriels--> + <suffix domain="aeroport.fr" /> + <suffix domain="assedic.fr" /> + <suffix domain="avocat.fr" /> + <suffix domain="avoues.fr" /> + <suffix domain="cci.fr" /> + <suffix domain="chambagri.fr" /> + <suffix domain="chirurgiens-dentistes.fr" /> + <suffix domain="experts-comptables.fr" /> + <suffix domain="geometre-expert.fr" /> + <suffix domain="gouv.fr" /> + <suffix domain="greta.fr" /> + <suffix domain="huissier-justice.fr" /> + <suffix domain="medecin.fr" /> + <suffix domain="notaires.fr" /> + <suffix domain="pharmacien.fr" /> + <suffix domain="port.fr" /> + <suffix domain="veterinaire.fr" /> + + <!-- ga : http://en.wikipedia.org/wiki/.ga--> + + <!-- gd : http://en.wikipedia.org/wiki/.gd--> + + <!-- ge : http://www.nic.net.ge/policy_en.pdf--> + <suffix domain="com.ge" /> + <suffix domain="edu.ge" /> + <suffix domain="gov.ge" /> + <suffix domain="org.ge" /> + <suffix domain="mil.ge" /> + <suffix domain="net.ge" /> + <suffix domain="pvt.ge" /> + + <!-- gf : http://en.wikipedia.org/wiki/.gf--> + + <!-- gg : http://www.channelisles.net/tandc.shtml--> + <suffix domain="co.gg" /> + <suffix domain="org.gg" /> + <suffix domain="net.gg" /> + <suffix domain="sch.gg" /> + <suffix domain="gov.gg" /> + + <!-- gh : http://www.ghana.com/domain.htm--> + + <!-- gi : http://www.nic.gi/rules.html--> + <suffix domain="com.gi" /> + <suffix domain="ltd.gi" /> + <suffix domain="gov.gi" /> + <suffix domain="mod.gi" /> + <suffix domain="edu.gi" /> + <suffix domain="org.gi" /> + + <!-- gl : http://en.wikipedia.org/wiki/.gl--> + + <!-- gm : http://www.nic.gm/htmlpages%5Cgm-policy.htm--> + + <!-- gn : http://psg.com/dns/gn/gn.txt--> + + <!-- gov : http://en.wikipedia.org/wiki/.gov--> + + <!-- gp : http://www.nic.gp/index_en.php?url=charte_en.php--> + <suffix domain="com.gp" /> + <suffix domain="net.gp" /> + <suffix domain="edu.gp" /> + <suffix domain="org.gp" /> + + <!-- gq : http://en.wikipedia.org/wiki/.gq--> + + <!-- gr : https://grweb.ics.forth.gr/english/1617-B-2002.html--> + <suffix domain="com.gr" /> + <suffix domain="edu.gr" /> + <suffix domain="net.gr" /> + <suffix domain="org.gr" /> + <suffix domain="gov.gr" /> + + <!-- gs : http://en.wikipedia.org/wiki/.gs--> + + <!-- gt : http://www.gt/politicas.html--> + + <!-- gu : http://gadao.gov.gu/registration.txt--> + + <!-- gw : http://en.wikipedia.org/wiki/.gw--> + + <!-- gy : http://en.wikipedia.org/wiki/.gy--> + + <!-- hk : http://en.wikipedia.org/wiki/.hk--> + <suffix domain="com.hk" /> + <suffix domain="edu.hk" /> + <suffix domain="gov.hk" /> + <suffix domain="idv.hk" /> + <suffix domain="net.hk" /> + <suffix domain="org.hk" /> + + <!-- hm : http://en.wikipedia.org/wiki/.hm--> + + <!-- hn : http://www.nic.hn/politicas/ps02,,05.html--> + <suffix domain="com.hn" /> + <suffix domain="edu.hn" /> + <suffix domain="org.hn" /> + <suffix domain="net.hn" /> + <suffix domain="mil.hn" /> + <suffix domain="gob.hn" /> + + <!-- hr : http://www.dns.hr/documents/pdf/HRTLD-regulations.pdf--> + <suffix domain="iz.hr" /> + <suffix domain="from.hr" /> + <suffix domain="name.hr" /> + <suffix domain="com.hr" /> + + <!-- ht : http://www.nic.ht/info/charte.cfm--> + <suffix domain="com.ht" /> + <suffix domain="shop.ht" /> + <suffix domain="firm.ht" /> + <suffix domain="info.ht" /> + <suffix domain="adult.ht" /> + <suffix domain="net.ht" /> + <suffix domain="pro.ht" /> + <suffix domain="org.ht" /> + <suffix domain="med.ht" /> + <suffix domain="art.ht" /> + <suffix domain="coop.ht" /> + <suffix domain="pol.ht" /> + <suffix domain="asso.ht" /> + <suffix domain="edu.ht" /> + <suffix domain="rel.ht" /> + <suffix domain="gouv.ht" /> + <suffix domain="perso.ht" /> + + <!-- hu : http://www.domain.hu/domain/English/sld.html--> + <suffix domain="co.hu" /> + <suffix domain="info.hu" /> + <suffix domain="org.hu" /> + <suffix domain="priv.hu" /> + <suffix domain="sport.hu" /> + <suffix domain="tm.hu" /> + <suffix domain="2000.hu" /> + <suffix domain="agrar.hu" /> + <suffix domain="bolt.hu" /> + <suffix domain="casino.hu" /> + <suffix domain="city.hu" /> + <suffix domain="erotica.hu" /> + <suffix domain="erotika.hu" /> + <suffix domain="film.hu" /> + <suffix domain="forum.hu" /> + <suffix domain="games.hu" /> + <suffix domain="hotel.hu" /> + <suffix domain="ingatlan.hu" /> + <suffix domain="jogasz.hu" /> + <suffix domain="konyvelo.hu" /> + <suffix domain="lakas.hu" /> + <suffix domain="media.hu" /> + <suffix domain="news.hu" /> + <suffix domain="reklam.hu" /> + <suffix domain="sex.hu" /> + <suffix domain="shop.hu" /> + <suffix domain="suli.hu" /> + <suffix domain="szex.hu" /> + <suffix domain="tozsde.hu" /> + <suffix domain="utazas.hu" /> + <suffix domain="video.hu" /> + + <!-- id : http://en.wikipedia.org/wiki/.id--> + + <!-- ie : http://en.wikipedia.org/wiki/.ie--> + + <!-- il : http://en.wikipedia.org/wiki/.il--> + + <!-- im : https://www.nic.im/pdfs/imfaqs.pdf--> + <suffix domain="co.im" /> + <suffix domain="ltd.co.im" /> + <suffix domain="plc.co.im" /> + <suffix domain="net.im" /> + <suffix domain="gov.im" /> + <suffix domain="org.im" /> + <suffix domain="nic.im" /> + <suffix domain="ac.im" /> + + <!-- in : http://en.wikipedia.org/wiki/.in--> + <suffix domain="co.in" /> + <suffix domain="firm.in" /> + <suffix domain="net.in" /> + <suffix domain="org.in" /> + <suffix domain="gen.in" /> + <suffix domain="ind.in" /> + <suffix domain="nic.in" /> + <suffix domain="ac.in" /> + <suffix domain="edu.in" /> + <suffix domain="res.in" /> + <suffix domain="gov.in" /> + <suffix domain="mil.in" /> + + <!-- info : http://en.wikipedia.org/wiki/.info--> + + <!-- int : http://en.wikipedia.org/wiki/.int--> + + <!-- io : http://www.nic.io/rules.html--> + <!-- list of 2nd level tlds ?--> + + <!-- iq : http://en.wikipedia.org/wiki/.iq--> + <!-- no registrar website found, but google shows .gov.iq and .edu.iq websites--> + <suffix domain="gov.iq" /> + <suffix domain="edu.iq" /> + + <!-- ir : http://www.nic.ir/ascii/Appendix1.htm--> + <suffix domain="ac.ir" /> + <suffix domain="co.ir" /> + <suffix domain="gov.ir" /> + <suffix domain="id.ir" /> + <suffix domain="net.ir" /> + <suffix domain="org.ir" /> + <suffix domain="sch.ir" /> + + <!-- is : http://www.isnic.is/domain/rules.php--> + <suffix domain="net.is" /> + <suffix domain="com.is" /> + <suffix domain="edu.is" /> + <suffix domain="gov.is" /> + <suffix domain="org.is" /> + <suffix domain="int.is" /> + + <!-- it : http://en.wikipedia.org/wiki/.it--> + <suffix domain="gov.edu" /> + <!-- geo-names found at http://www.nic.it/RA/en/domini/regole/nomi-riservati.pdf--> + <suffix domain="agrigento.it" /> + <suffix domain="ag.it" /> + <suffix domain="alessandria.it" /> + <suffix domain="al.it" /> + <suffix domain="ancona.it" /> + <suffix domain="an.it" /> + <suffix domain="aosta.it" /> + <suffix domain="aoste.it" /> + <suffix domain="ao.it" /> + <suffix domain="arezzo.it" /> + <suffix domain="ar.it" /> + <suffix domain="ascoli-piceno.it" /> + <suffix domain="ascolipiceno.it" /> + <suffix domain="ap.it" /> + <suffix domain="asti.it" /> + <suffix domain="at.it" /> + <suffix domain="avellino.it" /> + <suffix domain="av.it" /> + <suffix domain="bari.it" /> + <suffix domain="ba.it" /> + <suffix domain="barlettaandriatrani.it" /> + <suffix domain="barletta-andria-trani.it" /> + <suffix domain="belluno.it" /> + <suffix domain="bl.it" /> + <suffix domain="benevento.it" /> + <suffix domain="bn.it" /> + <suffix domain="bergamo.it" /> + <suffix domain="bg.it" /> + <suffix domain="biella.it" /> + <suffix domain="bi.it" /> + <suffix domain="bologna.it" /> + <suffix domain="bo.it" /> + <suffix domain="bolzano.it" /> + <suffix domain="bozen.it" /> + <suffix domain="balsan.it" /> + <suffix domain="alto-adige.it" /> + <suffix domain="altoadige.it" /> + <suffix domain="suedtirol.it" /> + <suffix domain="bz.it" /> + <suffix domain="brescia.it" /> + <suffix domain="bs.it" /> + <suffix domain="brindisi.it" /> + <suffix domain="br.it" /> + <suffix domain="cagliari.it" /> + <suffix domain="ca.it" /> + <suffix domain="caltanissetta.it" /> + <suffix domain="cl.it" /> + <suffix domain="campobasso.it" /> + <suffix domain="cb.it" /> + <suffix domain="caserta.it" /> + <suffix domain="ce.it" /> + <suffix domain="catania.it" /> + <suffix domain="ct.it" /> + <suffix domain="catanzaro.it" /> + <suffix domain="cz.it" /> + <suffix domain="chieti.it" /> + <suffix domain="ch.it" /> + <suffix domain="como.it" /> + <suffix domain="co.it" /> + <suffix domain="cosenza.it" /> + <suffix domain="cs.it" /> + <suffix domain="cremona.it" /> + <suffix domain="cr.it" /> + <suffix domain="crotone.it" /> + <suffix domain="kr.it" /> + <suffix domain="cuneo.it" /> + <suffix domain="cn.it" /> + <suffix domain="enna.it" /> + <suffix domain="en.it" /> + <suffix domain="fermo.it" /> + <suffix domain="ferrara.it" /> + <suffix domain="fe.it" /> + <suffix domain="firenze.it" /> + <suffix domain="florence.it" /> + <suffix domain="fi.it" /> + <suffix domain="foggia.it" /> + <suffix domain="fg.it" /> + <suffix domain="forli-cesena.it" /> + <suffix domain="forlicesena.it" /> + <suffix domain="fc.it" /> + <suffix domain="frosinone.it" /> + <suffix domain="fr.it" /> + <suffix domain="genova.it" /> + <suffix domain="genoa.it" /> + <suffix domain="ge.it" /> + <suffix domain="gorizia.it" /> + <suffix domain="go.it" /> + <suffix domain="grosseto.it" /> + <suffix domain="gr.it" /> + <suffix domain="imperia.it" /> + <suffix domain="im.it" /> + <suffix domain="isernia.it" /> + <suffix domain="is.it" /> + <suffix domain="laquila.it" /> + <suffix domain="aquila.it" /> + <suffix domain="aq.it" /> + <suffix domain="la-spezia.it" /> + <suffix domain="laspezia.it" /> + <suffix domain="sp.it" /> + <suffix domain="latina.it" /> + <suffix domain="lt.it" /> + <suffix domain="lecce.it" /> + <suffix domain="le.it" /> + <suffix domain="lecco.it" /> + <suffix domain="lc.it" /> + <suffix domain="livorno.it" /> + <suffix domain="li.it" /> + <suffix domain="lodi.it" /> + <suffix domain="lo.it" /> + <suffix domain="lucca.it" /> + <suffix domain="lu.it" /> + <suffix domain="macerata.it" /> + <suffix domain="mc.it" /> + <suffix domain="mantova.it" /> + <suffix domain="mn.it" /> + <suffix domain="massa-carrara.it" /> + <suffix domain="massacarrara.it" /> + <suffix domain="ms.it" /> + <suffix domain="matera.it" /> + <suffix domain="mt.it" /> + <suffix domain="messina.it" /> + <suffix domain="me.it" /> + <suffix domain="milano.it" /> + <suffix domain="milan.it" /> + <suffix domain="mi.it" /> + <suffix domain="modena.it" /> + <suffix domain="mo.it" /> + <suffix domain="monza.it" /> + <suffix domain="napoli.it" /> + <suffix domain="naples.it" /> + <suffix domain="na.it" /> + <suffix domain="novara.it" /> + <suffix domain="no.it" /> + <suffix domain="nuoro.it" /> + <suffix domain="nu.it" /> + <suffix domain="oristano.it" /> + <suffix domain="or.it" /> + <suffix domain="padova.it" /> + <suffix domain="padua.it" /> + <suffix domain="pd.it" /> + <suffix domain="palermo.it" /> + <suffix domain="pa.it" /> + <suffix domain="parma.it" /> + <suffix domain="pr.it" /> + <suffix domain="pavia.it" /> + <suffix domain="pv.it" /> + <suffix domain="perugia.it" /> + <suffix domain="pg.it" /> + <suffix domain="pescara.it" /> + <suffix domain="pe.it" /> + <suffix domain="pesaro-urbino.it" /> + <suffix domain="pesarourbino.it" /> + <suffix domain="pu.it" /> + <suffix domain="piacenza.it" /> + <suffix domain="pc.it" /> + <suffix domain="pisa.it" /> + <suffix domain="pi.it" /> + <suffix domain="pistoia.it" /> + <suffix domain="pt.it" /> + <suffix domain="pordenone.it" /> + <suffix domain="pn.it" /> + <suffix domain="potenza.it" /> + <suffix domain="pz.it" /> + <suffix domain="prato.it" /> + <suffix domain="po.it" /> + <suffix domain="ragusa.it" /> + <suffix domain="rg.it" /> + <suffix domain="ravenna.it" /> + <suffix domain="ra.it" /> + <suffix domain="reggio-calabria.it" /> + <suffix domain="reggiocalabria.it" /> + <suffix domain="rc.it" /> + <suffix domain="reggio-emilia.it" /> + <suffix domain="reggioemilia.it" /> + <suffix domain="re.it" /> + <suffix domain="rieti.it" /> + <suffix domain="ri.it" /> + <suffix domain="rimini.it" /> + <suffix domain="rn.it" /> + <suffix domain="roma.it" /> + <suffix domain="rome.it" /> + <suffix domain="rm.it" /> + <suffix domain="rovigo.it" /> + <suffix domain="ro.it" /> + <suffix domain="salerno.it" /> + <suffix domain="sa.it" /> + <suffix domain="sassari.it" /> + <suffix domain="ss.it" /> + <suffix domain="savona.it" /> + <suffix domain="sv.it" /> + <suffix domain="siena.it" /> + <suffix domain="si.it" /> + <suffix domain="siracusa.it" /> + <suffix domain="sr.it" /> + <suffix domain="sondrio.it" /> + <suffix domain="so.it" /> + <suffix domain="taranto.it" /> + <suffix domain="ta.it" /> + <suffix domain="teramo.it" /> + <suffix domain="te.it" /> + <suffix domain="terni.it" /> + <suffix domain="tr.it" /> + <suffix domain="torino.it" /> + <suffix domain="turin.it" /> + <suffix domain="to.it" /> + <suffix domain="trapani.it" /> + <suffix domain="tp.it" /> + <suffix domain="trento.it" /> + <suffix domain="trentino.it" /> + <suffix domain="tn.it" /> + <suffix domain="treviso.it" /> + <suffix domain="tv.it" /> + <suffix domain="trieste.it" /> + <suffix domain="ts.it" /> + <suffix domain="udine.it" /> + <suffix domain="ud.it" /> + <suffix domain="varese.it" /> + <suffix domain="va.it" /> + <suffix domain="venezia.it" /> + <suffix domain="venice.it" /> + <suffix domain="ve.it" /> + <suffix domain="verbania.it" /> + <suffix domain="vb.it" /> + <suffix domain="vercelli.it" /> + <suffix domain="vc.it" /> + <suffix domain="verona.it" /> + <suffix domain="vr.it" /> + <suffix domain="vibo-valentia.it" /> + <suffix domain="vibovalentia.it" /> + <suffix domain="vv.it" /> + <suffix domain="vicenza.it" /> + <suffix domain="vi.it" /> + <suffix domain="viterbo.it" /> + <suffix domain="vt.it" /> + + <!-- je : http://www.channelisles.net/tandc.shtml--> + <suffix domain="co.je" /> + <suffix domain="org.je" /> + <suffix domain="net.je" /> + <suffix domain="sch.je" /> + <suffix domain="gov.je" /> + + <!-- jm : http://www.com.jm/register.html--> + + <!-- jo : http://www.nis.gov.jo/dns/reg.html--> + <suffix domain="com.jo" /> + <suffix domain="org.jo" /> + <suffix domain="net.jo" /> + <suffix domain="edu.jo" /> + <suffix domain="gov.jo" /> + <suffix domain="mil.jo" /> + <suffix domain="myname.jo" /> + + <!-- jobs : http://en.wikipedia.org/wiki/.jobs--> + + <!-- jp : http://en.wikipedia.org/wiki/.jp--> + <suffix domain="ac.jp" /> + <suffix domain="ad.jp" /> + <suffix domain="co.jp" /> + <suffix domain="ed.jp" /> + <suffix domain="go.jp" /> + <suffix domain="gr.jp" /> + <suffix domain="lg.jp" /> + <suffix domain="ne.jp" /> + <suffix domain="or.jp" /> + <!-- jp geographical names--> + <!-- I can't find an official english explanantrion, but used https://bugzilla.mozilla.org/show_bug.cgi?id=252342#c31--> + <suffix domain="aichi.jp" /> + <suffix domain="akita.jp" /> + <suffix domain="aomori.jp" /> + <suffix domain="chiba.jp" /> + <suffix domain="ehime.jp" /> + <suffix domain="fukui.jp" /> + <suffix domain="fukuoka.jp" /> + <suffix domain="fukushima.jp" /> + <suffix domain="gifu.jp" /> + <suffix domain="gunma.jp" /> + <suffix domain="hiroshima.jp" /> + <suffix domain="hokkaido.jp" /> + <suffix domain="hyogo.jp" /> + <suffix domain="ibaraki.jp" /> + <suffix domain="ishikawa.jp" /> + <suffix domain="iwate.jp" /> + <suffix domain="kagawa.jp" /> + <suffix domain="kagoshima.jp" /> + <suffix domain="kanagawa.jp" /> + <suffix domain="kawasaki.jp" /> + <suffix domain="kitakyushu.jp" /> + <suffix domain="kobe.jp" /> + <suffix domain="kochi.jp" /> + <suffix domain="kumamoto.jp" /> + <suffix domain="kyoto.jp" /> + <suffix domain="mie.jp" /> + <suffix domain="miyagi.jp" /> + <suffix domain="miyazaki.jp" /> + <suffix domain="nagano.jp" /> + <suffix domain="nagasaki.jp" /> + <suffix domain="nagoya.jp" /> + <suffix domain="nara.jp" /> + <suffix domain="niigata.jp" /> + <suffix domain="oita.jp" /> + <suffix domain="okayama.jp" /> + <suffix domain="okinawa.jp" /> + <suffix domain="osaka.jp" /> + <suffix domain="saga.jp" /> + <suffix domain="saitama.jp" /> + <suffix domain="sapporo.jp" /> + <suffix domain="sendai.jp" /> + <suffix domain="shiga.jp" /> + <suffix domain="shimane.jp" /> + <suffix domain="shizuoka.jp" /> + <suffix domain="tochigi.jp" /> + <suffix domain="tokushima.jp" /> + <suffix domain="tokyo.jp" /> + <suffix domain="tottori.jp" /> + <suffix domain="toyama.jp" /> + <suffix domain="wakayama.jp" /> + <suffix domain="yamagata.jp" /> + <suffix domain="yamaguchi.jp" /> + <suffix domain="yamanashi.jp" /> + <suffix domain="yokohama.jp" /> + <suffix domain="metro.tokyo.jp" /> + <suffix domain="pref.aichi.jp" /> + <suffix domain="pref.akita.jp" /> + <suffix domain="pref.aomori.jp" /> + <suffix domain="pref.chiba.jp" /> + <suffix domain="pref.ehime.jp" /> + <suffix domain="pref.fukui.jp" /> + <suffix domain="pref.fukuoka.jp" /> + <suffix domain="pref.fukushima.jp" /> + <suffix domain="pref.gifu.jp" /> + <suffix domain="pref.gunma.jp" /> + <suffix domain="pref.hiroshima.jp" /> + <suffix domain="pref.hokkaido.jp" /> + <suffix domain="pref.hyogo.jp" /> + <suffix domain="pref.ibaraki.jp" /> + <suffix domain="pref.ishikawa.jp" /> + <suffix domain="pref.iwate.jp" /> + <suffix domain="pref.kagawa.jp" /> + <suffix domain="pref.kagoshima.jp" /> + <suffix domain="pref.kanagawa.jp" /> + <suffix domain="pref.kochi.jp" /> + <suffix domain="pref.kumamoto.jp" /> + <suffix domain="pref.kyoto.jp" /> + <suffix domain="pref.mie.jp" /> + <suffix domain="pref.miyagi.jp" /> + <suffix domain="pref.miyazaki.jp" /> + <suffix domain="pref.nagano.jp" /> + <suffix domain="pref.nagasaki.jp" /> + <suffix domain="pref.nara.jp" /> + <suffix domain="pref.niigata.jp" /> + <suffix domain="pref.oita.jp" /> + <suffix domain="pref.okayama.jp" /> + <suffix domain="pref.okinawa.jp" /> + <suffix domain="pref.osaka.jp" /> + <suffix domain="pref.saga.jp" /> + <suffix domain="pref.saitama.jp" /> + <suffix domain="pref.shiga.jp" /> + <suffix domain="pref.shimane.jp" /> + <suffix domain="pref.shizuoka.jp" /> + <suffix domain="pref.tochigi.jp" /> + <suffix domain="pref.tokushima.jp" /> + <suffix domain="pref.tottori.jp" /> + <suffix domain="pref.toyama.jp" /> + <suffix domain="pref.wakayama.jp" /> + <suffix domain="pref.yamagata.jp" /> + <suffix domain="pref.yamaguchi.jp" /> + <suffix domain="pref.yamanashi.jp" /> + <suffix domain="city.chiba.jp" /> + <suffix domain="city.fukuoka.jp" /> + <suffix domain="city.hiroshima.jp" /> + <suffix domain="city.kawasaki.jp" /> + <suffix domain="city.kitakyushu.jp" /> + <suffix domain="city.kobe.jp" /> + <suffix domain="city.kyoto.jp" /> + <suffix domain="city.nagoya.jp" /> + <suffix domain="city.osaka.jp" /> + <suffix domain="city.saitama.jp" /> + <suffix domain="city.sapporo.jp" /> + <suffix domain="city.sendai.jp" /> + <suffix domain="city.shizuoka.jp" /> + <suffix domain="city.yokohama.jp" /> + + <!-- ke : http://www.kenic.or.ke/index.php?option=com_content&task=view&id=117&Itemid=145--> + + <!-- kg : http://www.domain.kg/dmn_n.html--> + <suffix domain="org.kg" /> + <suffix domain="net.kg" /> + <suffix domain="com.kg" /> + <suffix domain="edu.kg" /> + <suffix domain="gov.kg" /> + <suffix domain="mil.kg" /> + + <!-- kh : http://www.mptc.gov.kh/dns_registration.htm--> + + <!-- ki : http://www.ki/dns/index.html--> + <suffix domain="edu.ki" /> + <suffix domain="biz.ki" /> + <suffix domain="net.ki" /> + <suffix domain="org.ki" /> + <suffix domain="gov.ki" /> + <suffix domain="info.ki" /> + <suffix domain="com.ki" /> + + <!-- km : http://en.wikipedia.org/wiki/.km--> + + <!-- kn : http://en.wikipedia.org/wiki/.kn--> + + <!-- kr : http://domain.nida.or.kr/eng/structure.jsp--> + <suffix domain="ac.kr" /> + <suffix domain="co.kr" /> + <suffix domain="go.kr" /> + <suffix domain="ne.kr" /> + <suffix domain="or.kr" /> + <suffix domain="re.kr" /> + <suffix domain="pe.kr" /> + <suffix domain="՜Ꞁ.kr" /> + + <!-- kw : http://en.wikipedia.org/wiki/.kw--> + + <!-- ky : http://www.icta.ky/da_ky_reg_dom.php--> + <suffix domain="edu.ky" /> + <suffix domain="gov.ky" /> + <suffix domain="com.ky" /> + <suffix domain="org.ky" /> + <suffix domain="net.ky" /> + + <!-- kz : http://en.wikipedia.org/wiki/.kz--> + <suffix domain="org.kz" /> + <suffix domain="edu.kz" /> + <suffix domain="net.kz" /> + <suffix domain="gov.kz" /> + <suffix domain="mil.kz" /> + <suffix domain="com.kz" /> + + <!-- la : http://en.wikipedia.org/wiki/.la--> + + <!-- lb : http://en.wikipedia.org/wiki/.lb--> + + <!-- lc : http://en.wikipedia.org/wiki/.lc--> + <suffix domain="com.lc" /> + <suffix domain="org.lc" /> + <suffix domain="edu.lc" /> + <suffix domain="gov.lc" /> + + <!-- li : http://en.wikipedia.org/wiki/.li--> + + <!-- lk : http://www.nic.lk/seclevpr.html--> + <suffix domain="gov.lk" /> + <suffix domain="sch.lk" /> + <suffix domain="net.lk" /> + <suffix domain="int.lk" /> + <suffix domain="com.lk" /> + <suffix domain="org.lk" /> + <suffix domain="edu.lk" /> + <suffix domain="ngo.lk" /> + <suffix domain="soc.lk" /> + <suffix domain="web.lk" /> + <suffix domain="ltd.lk" /> + <suffix domain="assn.lk" /> + <suffix domain="grp.lk" /> + <suffix domain="hotel.lk" /> + + <!-- lr : http://psg.com/dns/lr/lr.txt--> + + <!-- ls : http://en.wikipedia.org/wiki/.ls--> + <suffix domain="co.ls" /> + <suffix domain="org.ls" /> + + <!-- lt : http://en.wikipedia.org/wiki/.lt--> + + <!-- lu : http://www.dns.lu/en/--> + + <!-- lv : http://www.nic.lv/DNS/En/generic.php--> + <suffix domain="com.lv" /> + <suffix domain="edu.lv" /> + <suffix domain="gov.lv" /> + <suffix domain="org.lv" /> + <suffix domain="mil.lv" /> + <suffix domain="id.lv" /> + <suffix domain="net.lv" /> + <suffix domain="asn.lv" /> + <suffix domain="conf.lv" /> + + <!-- ly : http://www.nic.ly/regulations.php--> + <suffix domain="com.ly" /> + <suffix domain="net.ly" /> + <suffix domain="gov.ly" /> + <suffix domain="plc.ly" /> + <suffix domain="edu.ly" /> + <suffix domain="sch.ly" /> + <suffix domain="med.ly" /> + <suffix domain="org.ly" /> + <suffix domain="id.ly" /> + + <!-- ma : http://en.wikipedia.org/wiki/.ma--> + <!-- list of 2nd level tlds ?--> + <suffix domain="co.ma" /> + <suffix domain="net.ma" /> + <suffix domain="gov.ma" /> + <suffix domain="org.ma" /> + + <!-- mc : http://www.nic.mc/--> + <suffix domain="tm.mc" /> + <suffix domain="asso.mc" /> + + <!-- md : http://en.wikipedia.org/wiki/.md--> + + <!-- mg : http://www.nic.mg/tarif.htm--> + <suffix domain="org.mg" /> + <suffix domain="nom.mg" /> + <suffix domain="gov.mg" /> + <suffix domain="prd.mg" /> + <suffix domain="tm.mg" /> + <suffix domain="edu.mg" /> + <suffix domain="mil.mg" /> + <suffix domain="com.mg" /> + + <!-- mh : http://en.wikipedia.org/wiki/.mh--> + + <!-- mil : http://en.wikipedia.org/wiki/.mil--> + + <!-- mk : http://en.wikipedia.org/wiki/.mk--> + <!-- list of 2nd level tlds ?--> + <suffix domain="com.mk" /> + <suffix domain="gov.mk" /> + <suffix domain="org.mk" /> + <suffix domain="net.mk" /> + <suffix domain="edu.mk" /> + + <!-- ml : http://www.gobin.info/domainname/ml-template.doc--> + + <!-- mm : http://en.wikipedia.org/wiki/.mm--> + + <!-- mn : http://en.wikipedia.org/wiki/.mn--> + <suffix domain="gov.mn" /> + <suffix domain="edu.mn" /> + <suffix domain="org.mn" /> + + <!-- mo : http://www.monic.net.mo/--> + <suffix domain="com.mo" /> + <suffix domain="net.mo" /> + <suffix domain="org.mo" /> + <suffix domain="edu.mo" /> + <suffix domain="gov.mo" /> + + <!-- mobi : http://en.wikipedia.org/wiki/.mobi--> + + <!-- mp : http://www.dot.mp/--> + + <!-- mq : http://en.wikipedia.org/wiki/.mq--> + + <!-- mr : http://en.wikipedia.org/wiki/.mr--> + + <!-- ms : http://en.wikipedia.org/wiki/.ms--> + + <!-- mt : https://www.nic.org.mt/dotmt/--> + + <!-- mu : http://en.wikipedia.org/wiki/.mu--> + <!-- list of 2nd level tlds ?--> + + <!-- museum : http://about.museum/naming/--> + <!-- there are 2nd-level TLD's, but there's no list--> + + <!-- mv : http://en.wikipedia.org/wiki/.mv--> + + <!-- mw : http://www.registrar.mw/--> + <suffix domain="ac.mw" /> + <suffix domain="biz.mw" /> + <suffix domain="co.mw" /> + <suffix domain="com.mw" /> + <suffix domain="coop.mw" /> + <suffix domain="edu.mw" /> + <suffix domain="gov.mw" /> + <suffix domain="int.mw" /> + <suffix domain="net.mw" /> + <suffix domain="org.mw" /> + + <!-- mx : http://www.nic.mx/--> + + <!-- my : http://www.mynic.net.my/--> + + <!-- mz : http://www.gobin.info/domainname/mz-template.doc--> + + <!-- na : http://www.na-nic.com.na/--> + <!-- list of 2nd level tlds ?--> + + <!-- name : has 2nd-level tlds, but there's no list of them--> + + <!-- nc : http://www.cctld.nc/--> + + <!-- ne : http://en.wikipedia.org/wiki/.ne--> + + <!-- net : http://en.wikipedia.org/wiki/.net--> + + <!-- nf : http://en.wikipedia.org/wiki/.nf--> + <suffix domain="com.nf" /> + <suffix domain="net.nf" /> + <suffix domain="per.nf" /> + <suffix domain="rec.nf" /> + <suffix domain="web.nf" /> + <suffix domain="arts.nf" /> + <suffix domain="firm.nf" /> + <suffix domain="info.nf" /> + <suffix domain="other.nf" /> + <suffix domain="store.nf" /> + + <!-- ng : http://psg.com/dns/ng/--> + + <!-- ni : http://www.nic.ni/dominios.htm--> + + <!-- nl : http://www.domain-registry.nl/ace.php/c,728,122,,,,Home.html--> + + <!-- no : http://www.norid.no/regelverk/index.en.html--> + <suffix domain="fhs.no" /> + <suffix domain="vgs.no" /> + <suffix domain="fylkesbibl.no" /> + <suffix domain="folkebibl.no" /> + <suffix domain="museum.no" /> + <suffix domain="idrett.no" /> + <suffix domain="mil.no" /> + <suffix domain="stat.no" /> + <suffix domain="dep.no" /> + <suffix domain="kommune.no" /> + <suffix domain="herad.no" /> + <suffix domain="priv.no" /> + <!-- no geographical names : http://www.norid.no/regelverk/vedlegg-b.en.html--> + <!-- counties--> + <suffix domain="aa.no" /> + <suffix domain="ah.no" /> + <suffix domain="bu.no" /> + <suffix domain="fm.no" /> + <suffix domain="hl.no" /> + <suffix domain="hm.no" /> + <suffix domain="jan-mayen.no" /> + <suffix domain="mr.no" /> + <suffix domain="nl.no" /> + <suffix domain="nt.no" /> + <suffix domain="of.no" /> + <suffix domain="ol.no" /> + <suffix domain="oslo.no" /> + <suffix domain="rl.no" /> + <suffix domain="sf.no" /> + <suffix domain="st.no" /> + <suffix domain="svalbard.no" /> + <suffix domain="tm.no" /> + <suffix domain="tr.no" /> + <suffix domain="va.no" /> + <suffix domain="vf.no" /> + <!-- primary and lower secondary schools per county--> + <suffix domain="gs.aa.no" /> + <suffix domain="gs.ah.no" /> + <suffix domain="gs.bu.no" /> + <suffix domain="gs.fm.no" /> + <suffix domain="gs.hl.no" /> + <suffix domain="gs.hm.no" /> + <suffix domain="gs.jan-mayen.no" /> + <suffix domain="gs.mr.no" /> + <suffix domain="gs.nl.no" /> + <suffix domain="gs.nt.no" /> + <suffix domain="gs.of.no" /> + <suffix domain="gs.ol.no" /> + <suffix domain="gs.oslo.no" /> + <suffix domain="gs.rl.no" /> + <suffix domain="gs.sf.no" /> + <suffix domain="gs.st.no" /> + <suffix domain="gs.svalbard.no" /> + <suffix domain="gs.tm.no" /> + <suffix domain="gs.tr.no" /> + <suffix domain="gs.va.no" /> + <suffix domain="gs.vf.no" /> + <!-- cities--> + <suffix domain="akrehamn.no" /> + <suffix domain="Ã¥krehamn.no" /> + <suffix domain="algard.no" /> + <suffix domain="Ã¥lgÃ¥rd.no" /> + <suffix domain="arna.no" /> + <suffix domain="brumunddal.no" /> + <suffix domain="bryne.no" /> + <suffix domain="bronnoysund.no" /> + <suffix domain="brÞnnÞysund.no" /> + <suffix domain="drobak.no" /> + <suffix domain="drÞbak.no" /> + <suffix domain="egersund.no" /> + <suffix domain="fetsund.no" /> + <suffix domain="floro.no" /> + <suffix domain="florÞ.no" /> + <suffix domain="fredrikstad.no" /> + <suffix domain="hokksund.no" /> + <suffix domain="honefoss.no" /> + <suffix domain="hÞnefoss.no" /> + <suffix domain="jessheim.no" /> + <suffix domain="jorpeland.no" /> + <suffix domain="jÞrpeland.no" /> + <suffix domain="kirkenes.no" /> + <suffix domain="kopervik.no" /> + <suffix domain="krokstadelva.no" /> + <suffix domain="langevag.no" /> + <suffix domain="langevÃ¥g.no" /> + <suffix domain="leirvik.no" /> + <suffix domain="mjondalen.no" /> + <suffix domain="mjÞndalen.no" /> + <suffix domain="mo-i-rana.no" /> + <suffix domain="mosjoen.no" /> + <suffix domain="mosjÞen.no" /> + <suffix domain="nesoddtangen.no" /> + <suffix domain="orkanger.no" /> + <suffix domain="osoyro.no" /> + <suffix domain="osÞyro.no" /> + <suffix domain="raholt.no" /> + <suffix domain="rÃ¥holt.no" /> + <suffix domain="sandnessjoen.no" /> + <suffix domain="sandnessjÞen.no" /> + <suffix domain="skedsmokorset.no" /> + <suffix domain="slattum.no" /> + <suffix domain="spjelkavik.no" /> + <suffix domain="stathelle.no" /> + <suffix domain="stavern.no" /> + <suffix domain="stjordalshalsen.no" /> + <suffix domain="stjÞrdalshalsen.no" /> + <suffix domain="tananger.no" /> + <suffix domain="tranby.no" /> + <suffix domain="vossevangen.no" /> + <!-- communities--> + <suffix domain="afjord.no" /> + <suffix domain="Ã¥fjord.no" /> + <suffix domain="agdenes.no" /> + <suffix domain="al.no" /> + <suffix domain="Ã¥l.no" /> + <suffix domain="alesund.no" /> + <suffix domain="Ã¥lesund.no" /> + <suffix domain="alstahaug.no" /> + <suffix domain="alta.no" /> + <suffix domain="áltá.no" /> + <suffix domain="alaheadju.no" /> + <suffix domain="álaheadju.no" /> + <suffix domain="alvdal.no" /> + <suffix domain="amli.no" /> + <suffix domain="Ã¥mli.no" /> + <suffix domain="amot.no" /> + <suffix domain="Ã¥mot.no" /> + <suffix domain="andebu.no" /> + <suffix domain="andoy.no" /> + <suffix domain="andÞy.no" /> + <suffix domain="andasuolo.no" /> + <suffix domain="ardal.no" /> + <suffix domain="Ã¥rdal.no" /> + <suffix domain="aremark.no" /> + <suffix domain="arendal.no" /> + <suffix domain="Ã¥s.no" /> + <suffix domain="aseral.no" /> + <suffix domain="Ã¥seral.no" /> + <suffix domain="asker.no" /> + <suffix domain="askim.no" /> + <suffix domain="askvoll.no" /> + <suffix domain="askoy.no" /> + <suffix domain="askÞy.no" /> + <suffix domain="asnes.no" /> + <suffix domain="Ã¥snes.no" /> + <suffix domain="audnedaln.no" /> + <suffix domain="aukra.no" /> + <suffix domain="aure.no" /> + <suffix domain="aurland.no" /> + <suffix domain="aurskog-holand.no" /> + <suffix domain="aurskog-hÞland.no" /> + <suffix domain="austevoll.no" /> + <suffix domain="austrheim.no" /> + <suffix domain="averoy.no" /> + <suffix domain="averÞy.no" /> + <suffix domain="balestrand.no" /> + <suffix domain="ballangen.no" /> + <suffix domain="balat.no" /> + <suffix domain="bálát.no" /> + <suffix domain="balsfjord.no" /> + <suffix domain="bahccavuotna.no" /> + <suffix domain="báhccavuotna.no" /> + <suffix domain="bamble.no" /> + <suffix domain="bardu.no" /> + <suffix domain="beardu.no" /> + <suffix domain="beiarn.no" /> + <suffix domain="bajddar.no" /> + <suffix domain="bájddar.no" /> + <suffix domain="baidar.no" /> + <suffix domain="báidár.no" /> + <suffix domain="berg.no" /> + <suffix domain="bergen.no" /> + <suffix domain="berlevag.no" /> + <suffix domain="berlevÃ¥g.no" /> + <suffix domain="bearalvahki.no" /> + <suffix domain="bearalváhki.no" /> + <suffix domain="bindal.no" /> + <suffix domain="birkenes.no" /> + <suffix domain="bjarkoy.no" /> + <suffix domain="bjarkÞy.no" /> + <suffix domain="bjerkreim.no" /> + <suffix domain="bjugn.no" /> + <suffix domain="bodo.no" /> + <suffix domain="bodÞ.no" /> + <suffix domain="badaddja.no" /> + <suffix domain="bÃ¥dÃ¥ddjÃ¥.no" /> + <suffix domain="budejju.no" /> + <suffix domain="bokn.no" /> + <suffix domain="bremanger.no" /> + <suffix domain="bronnoy.no" /> + <suffix domain="brÞnnÞy.no" /> + <suffix domain="bygland.no" /> + <suffix domain="bykle.no" /> + <suffix domain="barum.no" /> + <suffix domain="bÊrum.no" /> + <suffix domain="bo.telemark.no" /> + <suffix domain="bÞ.telemark.no" /> + <suffix domain="bo.nordland.no" /> + <suffix domain="bÞ.nordland.no" /> + <suffix domain="bievat.no" /> + <suffix domain="bievát.no" /> + <suffix domain="bomlo.no" /> + <suffix domain="bÞmlo.no" /> + <suffix domain="batsfjord.no" /> + <suffix domain="bÃ¥tsfjord.no" /> + <suffix domain="bahcavuotna.no" /> + <suffix domain="báhcavuotna.no" /> + <suffix domain="dovre.no" /> + <suffix domain="drammen.no" /> + <suffix domain="drangedal.no" /> + <suffix domain="dyroy.no" /> + <suffix domain="dyrÞy.no" /> + <suffix domain="donna.no" /> + <suffix domain="dÞnna.no" /> + <suffix domain="eid.no" /> + <suffix domain="eidfjord.no" /> + <suffix domain="eidsberg.no" /> + <suffix domain="eidskog.no" /> + <suffix domain="eidsvoll.no" /> + <suffix domain="eigersund.no" /> + <suffix domain="elverum.no" /> + <suffix domain="enebakk.no" /> + <suffix domain="engerdal.no" /> + <suffix domain="etne.no" /> + <suffix domain="etnedal.no" /> + <suffix domain="evenes.no" /> + <suffix domain="evenassi.no" /> + <suffix domain="evenášši.no" /> + <suffix domain="evje-og-hornnes.no" /> + <suffix domain="farsund.no" /> + <suffix domain="fauske.no" /> + <suffix domain="fuossko.no" /> + <suffix domain="fuoisku.no" /> + <suffix domain="fedje.no" /> + <suffix domain="fet.no" /> + <suffix domain="finnoy.no" /> + <suffix domain="finnÞy.no" /> + <suffix domain="fitjar.no" /> + <suffix domain="fjaler.no" /> + <suffix domain="fjell.no" /> + <suffix domain="flakstad.no" /> + <suffix domain="flatanger.no" /> + <suffix domain="flekkefjord.no" /> + <suffix domain="flesberg.no" /> + <suffix domain="flora.no" /> + <suffix domain="fla.no" /> + <suffix domain="flÃ¥.no" /> + <suffix domain="folldal.no" /> + <suffix domain="forsand.no" /> + <suffix domain="fosnes.no" /> + <suffix domain="fredrikstad.no" /> + <suffix domain="frei.no" /> + <suffix domain="frogn.no" /> + <suffix domain="froland.no" /> + <suffix domain="frosta.no" /> + <suffix domain="frana.no" /> + <suffix domain="frÊna.no" /> + <suffix domain="froya.no" /> + <suffix domain="frÞya.no" /> + <suffix domain="fusa.no" /> + <suffix domain="fyresdal.no" /> + <suffix domain="forde.no" /> + <suffix domain="fÞrde.no" /> + <suffix domain="gamvik.no" /> + <suffix domain="gangaviika.no" /> + <suffix domain="gaular.no" /> + <suffix domain="gausdal.no" /> + <suffix domain="gildeskal.no" /> + <suffix domain="gildeskÃ¥l.no" /> + <suffix domain="giske.no" /> + <suffix domain="gjemnes.no" /> + <suffix domain="gjerdrum.no" /> + <suffix domain="gjerstad.no" /> + <suffix domain="gjesdal.no" /> + <suffix domain="gjovik.no" /> + <suffix domain="gjÞvik.no" /> + <suffix domain="gloppen.no" /> + <suffix domain="gol.no" /> + <suffix domain="gran.no" /> + <suffix domain="grane.no" /> + <suffix domain="granvin.no" /> + <suffix domain="gratangen.no" /> + <suffix domain="grimstad.no" /> + <suffix domain="grong.no" /> + <suffix domain="kraanghke.no" /> + <suffix domain="krÃ¥anghke.no" /> + <suffix domain="grue.no" /> + <suffix domain="gulen.no" /> + <suffix domain="hadsel.no" /> + <suffix domain="halden.no" /> + <suffix domain="halsa.no" /> + <suffix domain="hamar.no" /> + <suffix domain="hamaroy.no" /> + <suffix domain="habmer.no" /> + <suffix domain="hábmer.no" /> + <suffix domain="hapmir.no" /> + <suffix domain="hápmir.no" /> + <suffix domain="hammerfest.no" /> + <suffix domain="hammarfeasta.no" /> + <suffix domain="hámmárfeasta.no" /> + <suffix domain="haram.no" /> + <suffix domain="hareid.no" /> + <suffix domain="harstad.no" /> + <suffix domain="hasvik.no" /> + <suffix domain="aknoluokta.no" /> + <suffix domain="ákŋoluokta.no" /> + <suffix domain="hattfjelldal.no" /> + <suffix domain="aarborte.no" /> + <suffix domain="haugesund.no" /> + <suffix domain="hemne.no" /> + <suffix domain="hemnes.no" /> + <suffix domain="hemsedal.no" /> + <suffix domain="heroy.more-og-romsdal.no" /> + <suffix domain="herÞy.mÞre-og-romsdal.no" /> + <suffix domain="heroy.nordland.no" /> + <suffix domain="herÞy.nordland.no" /> + <suffix domain="hitra.no" /> + <suffix domain="hjartdal.no" /> + <suffix domain="hjelmeland.no" /> + <suffix domain="hobol.no" /> + <suffix domain="hobÞl.no" /> + <suffix domain="hof.no" /> + <suffix domain="hol.no" /> + <suffix domain="hole.no" /> + <suffix domain="holmestrand.no" /> + <suffix domain="holtalen.no" /> + <suffix domain="holtÃ¥len.no" /> + <suffix domain="hornindal.no" /> + <suffix domain="horten.no" /> + <suffix domain="hurdal.no" /> + <suffix domain="hurum.no" /> + <suffix domain="hvaler.no" /> + <suffix domain="hyllestad.no" /> + <suffix domain="hagebostad.no" /> + <suffix domain="hÊgebostad.no" /> + <suffix domain="hoyanger.no" /> + <suffix domain="hÞyanger.no" /> + <suffix domain="hoylandet.no" /> + <suffix domain="hÞylandet.no" /> + <suffix domain="ha.no" /> + <suffix domain="hÃ¥.no" /> + <suffix domain="ibestad.no" /> + <suffix domain="inderoy.no" /> + <suffix domain="inderÞy.no" /> + <suffix domain="iveland.no" /> + <suffix domain="jevnaker.no" /> + <suffix domain="jondal.no" /> + <suffix domain="jolster.no" /> + <suffix domain="jÞlster.no" /> + <suffix domain="karasjok.no" /> + <suffix domain="karasjohka.no" /> + <suffix domain="kárášjohka.no" /> + <suffix domain="karlsoy.no" /> + <suffix domain="galsa.no" /> + <suffix domain="gálsá.no" /> + <suffix domain="karmoy.no" /> + <suffix domain="karmÞy.no" /> + <suffix domain="kautokeino.no" /> + <suffix domain="guovdageaidnu.no" /> + <suffix domain="klepp.no" /> + <suffix domain="klabu.no" /> + <suffix domain="klÊbu.no" /> + <suffix domain="kongsberg.no" /> + <suffix domain="kongsvinger.no" /> + <suffix domain="kragero.no" /> + <suffix domain="kragerÞ.no" /> + <suffix domain="kristiansand.no" /> + <suffix domain="kristiansund.no" /> + <suffix domain="krodsherad.no" /> + <suffix domain="krÞdsherad.no" /> + <suffix domain="kvalsund.no" /> + <suffix domain="rahkkeravju.no" /> + <suffix domain="ráhkkerávju.no" /> + <suffix domain="kvam.no" /> + <suffix domain="kvinesdal.no" /> + <suffix domain="kvinnherad.no" /> + <suffix domain="kviteseid.no" /> + <suffix domain="kvitsoy.no" /> + <suffix domain="kvitsÞy.no" /> + <suffix domain="kvafjord.no" /> + <suffix domain="kvÊfjord.no" /> + <suffix domain="giehtavuoatna.no" /> + <suffix domain="kvanangen.no" /> + <suffix domain="kvÊnangen.no" /> + <suffix domain="navuotna.no" /> + <suffix domain="návuotna.no" /> + <suffix domain="kafjord.no" /> + <suffix domain="kÃ¥fjord.no" /> + <suffix domain="gaivuotna.no" /> + <suffix domain="gáivuotna.no" /> + <suffix domain="larvik.no" /> + <suffix domain="lavangen.no" /> + <suffix domain="lavagis.no" /> + <suffix domain="loabat.no" /> + <suffix domain="loabát.no" /> + <suffix domain="lebesby.no" /> + <suffix domain="davvesiida.no" /> + <suffix domain="leikanger.no" /> + <suffix domain="leirfjord.no" /> + <suffix domain="leka.no" /> + <suffix domain="leksvik.no" /> + <suffix domain="lenvik.no" /> + <suffix domain="leangaviika.no" /> + <suffix domain="lea?gaviika.no" /> + <suffix domain="lesja.no" /> + <suffix domain="levanger.no" /> + <suffix domain="lier.no" /> + <suffix domain="lierne.no" /> + <suffix domain="lillehammer.no" /> + <suffix domain="lillesand.no" /> + <suffix domain="lindesnes.no" /> + <suffix domain="lindas.no" /> + <suffix domain="lindÃ¥s.no" /> + <suffix domain="lom.no" /> + <suffix domain="loppa.no" /> + <suffix domain="lahppi.no" /> + <suffix domain="láhppi.no" /> + <suffix domain="lund.no" /> + <suffix domain="lunner.no" /> + <suffix domain="luroy.no" /> + <suffix domain="lurÞy.no" /> + <suffix domain="luster.no" /> + <suffix domain="lyngdal.no" /> + <suffix domain="lyngen.no" /> + <suffix domain="ivgu.no" /> + <suffix domain="lardal.no" /> + <suffix domain="lerdal.no" /> + <suffix domain="lÊrdal.no" /> + <suffix domain="lodingen.no" /> + <suffix domain="lÞdingen.no" /> + <suffix domain="lorenskog.no" /> + <suffix domain="lÞrenskog.no" /> + <suffix domain="loten.no" /> + <suffix domain="lÞten.no" /> + <suffix domain="malvik.no" /> + <suffix domain="masoy.no" /> + <suffix domain="mÃ¥sÞy.no" /> + <suffix domain="muosat.no" /> + <suffix domain="muosát.no" /> + <suffix domain="mandal.no" /> + <suffix domain="marker.no" /> + <suffix domain="marnardal.no" /> + <suffix domain="masfjorden.no" /> + <suffix domain="meland.no" /> + <suffix domain="meldal.no" /> + <suffix domain="melhus.no" /> + <suffix domain="meloy.no" /> + <suffix domain="melÞy.no" /> + <suffix domain="meraker.no" /> + <suffix domain="merÃ¥ker.no" /> + <suffix domain="moareke.no" /> + <suffix domain="moÃ¥reke.no" /> + <suffix domain="midsund.no" /> + <suffix domain="midtre-gauldal.no" /> + <suffix domain="modalen.no" /> + <suffix domain="modum.no" /> + <suffix domain="molde.no" /> + <suffix domain="moskenes.no" /> + <suffix domain="moss.no" /> + <suffix domain="mosvik.no" /> + <suffix domain="malselv.no" /> + <suffix domain="mÃ¥lselv.no" /> + <suffix domain="malatvuopmi.no" /> + <suffix domain="málatvuopmi.no" /> + <suffix domain="namdalseid.no" /> + <suffix domain="aejrie.no" /> + <suffix domain="namsos.no" /> + <suffix domain="namsskogan.no" /> + <suffix domain="naamesjevuemie.no" /> + <suffix domain="nååmesjevuemie.no" /> + <suffix domain="laakesvuemie.no" /> + <suffix domain="nannestad.no" /> + <suffix domain="narvik.no" /> + <suffix domain="narviika.no" /> + <suffix domain="naustdal.no" /> + <suffix domain="nedre-eiker.no" /> + <suffix domain="nes.akershus.no" /> + <suffix domain="nes.buskerud.no" /> + <suffix domain="nesna.no" /> + <suffix domain="nesodden.no" /> + <suffix domain="nesseby.no" /> + <suffix domain="unjarga.no" /> + <suffix domain="unjárga.no" /> + <suffix domain="nesset.no" /> + <suffix domain="nissedal.no" /> + <suffix domain="nittedal.no" /> + <suffix domain="nord-aurdal.no" /> + <suffix domain="nord-fron.no" /> + <suffix domain="nord-odal.no" /> + <suffix domain="norddal.no" /> + <suffix domain="nordkapp.no" /> + <suffix domain="davvenjarga.no" /> + <suffix domain="davvenjárga.no" /> + <suffix domain="nordre-land.no" /> + <suffix domain="nordreisa.no" /> + <suffix domain="raisa.no" /> + <suffix domain="ráisa.no" /> + <suffix domain="nore-og-uvdal.no" /> + <suffix domain="notodden.no" /> + <suffix domain="naroy.no" /> + <suffix domain="nÊrÞy.no" /> + <suffix domain="notteroy.no" /> + <suffix domain="nÞtterÞy.no" /> + <suffix domain="odda.no" /> + <suffix domain="oksnes.no" /> + <suffix domain="`Þksnes.no" /> + <suffix domain="oppdal.no" /> + <suffix domain="oppegard.no" /> + <suffix domain="oppegÃ¥rd.no" /> + <suffix domain="orkdal.no" /> + <suffix domain="orland.no" /> + <suffix domain="Þrland.no" /> + <suffix domain="orskog.no" /> + <suffix domain="Þrskog.no" /> + <suffix domain="orsta.no" /> + <suffix domain="Þrsta.no" /> + <suffix domain="os.hedmark.no" /> + <suffix domain="os.hordaland.no" /> + <suffix domain="osen.no" /> + <suffix domain="osteroy.no" /> + <suffix domain="osterÞy.no" /> + <suffix domain="ostre-toten.no" /> + <suffix domain="Þstre-toten.no" /> + <suffix domain="overhalla.no" /> + <suffix domain="ovre-eiker.no" /> + <suffix domain="Þvre-eiker.no" /> + <suffix domain="oyer.no" /> + <suffix domain="Þyer.no" /> + <suffix domain="oygarden.no" /> + <suffix domain="Þygarden.no" /> + <suffix domain="oystre-slidre.no" /> + <suffix domain="Þystre-slidre.no" /> + <suffix domain="porsanger.no" /> + <suffix domain="porsangu.no" /> + <suffix domain="porsáŋgu.no" /> + <suffix domain="porsgrunn.no" /> + <suffix domain="radoy.no" /> + <suffix domain="radÞy.no" /> + <suffix domain="rakkestad.no" /> + <suffix domain="rana.no" /> + <suffix domain="ruovat.no" /> + <suffix domain="randaberg.no" /> + <suffix domain="rauma.no" /> + <suffix domain="rendalen.no" /> + <suffix domain="rennebu.no" /> + <suffix domain="rennesoy.no" /> + <suffix domain="rennesÞy.no" /> + <suffix domain="rindal.no" /> + <suffix domain="ringebu.no" /> + <suffix domain="ringerike.no" /> + <suffix domain="ringsaker.no" /> + <suffix domain="rissa.no" /> + <suffix domain="risor.no" /> + <suffix domain="risÞr.no" /> + <suffix domain="roan.no" /> + <suffix domain="rollag.no" /> + <suffix domain="rygge.no" /> + <suffix domain="ralingen.no" /> + <suffix domain="rÊlingen.no" /> + <suffix domain="rodoy.no" /> + <suffix domain="rÞdÞy.no" /> + <suffix domain="romskog.no" /> + <suffix domain="rÞmskog.no" /> + <suffix domain="roros.no" /> + <suffix domain="rÞros.no" /> + <suffix domain="rost.no" /> + <suffix domain="rÞst.no" /> + <suffix domain="royken.no" /> + <suffix domain="rÞyken.no" /> + <suffix domain="royrvik.no" /> + <suffix domain="rÞyrvik.no" /> + <suffix domain="rade.no" /> + <suffix domain="rÃ¥de.no" /> + <suffix domain="salangen.no" /> + <suffix domain="siellak.no" /> + <suffix domain="saltdal.no" /> + <suffix domain="sálát.no" /> + <suffix domain="sálat.no" /> + <suffix domain="samnanger.no" /> + <suffix domain="sande.more-og-romsdal.no" /> + <suffix domain="sande.mÞre-og-romsdal.no" /> + <suffix domain="sande.vestfold.no" /> + <suffix domain="sandefjord.no" /> + <suffix domain="sandnes.no" /> + <suffix domain="sandoy.no" /> + <suffix domain="sandÞy.no" /> + <suffix domain="sarpsborg.no" /> + <suffix domain="sauda.no" /> + <suffix domain="sauherad.no" /> + <suffix domain="sel.no" /> + <suffix domain="selbu.no" /> + <suffix domain="selje.no" /> + <suffix domain="seljord.no" /> + <suffix domain="sigdal.no" /> + <suffix domain="siljan.no" /> + <suffix domain="sirdal.no" /> + <suffix domain="skaun.no" /> + <suffix domain="skedsmo.no" /> + <suffix domain="ski.no" /> + <suffix domain="skien.no" /> + <suffix domain="skiptvet.no" /> + <suffix domain="skjervoy.no" /> + <suffix domain="skjervÞy.no" /> + <suffix domain="skierva.no" /> + <suffix domain="skiervá.no" /> + <suffix domain="skjak.no" /> + <suffix domain="skjÃ¥k.no" /> + <suffix domain="skodje.no" /> + <suffix domain="skanland.no" /> + <suffix domain="skÃ¥nland.no" /> + <suffix domain="skanit.no" /> + <suffix domain="skánit.no" /> + <suffix domain="smola.no" /> + <suffix domain="smÞla.no" /> + <suffix domain="snillfjord.no" /> + <suffix domain="snasa.no" /> + <suffix domain="snÃ¥sa.no" /> + <suffix domain="snoasa.no" /> + <suffix domain="snaase.no" /> + <suffix domain="snÃ¥ase.no" /> + <suffix domain="sogndal.no" /> + <suffix domain="sokndal.no" /> + <suffix domain="sola.no" /> + <suffix domain="solund.no" /> + <suffix domain="songdalen.no" /> + <suffix domain="sortland.no" /> + <suffix domain="spydeberg.no" /> + <suffix domain="stange.no" /> + <suffix domain="stavanger.no" /> + <suffix domain="steigen.no" /> + <suffix domain="steinkjer.no" /> + <suffix domain="stjordal.no" /> + <suffix domain="stjÞrdal.no" /> + <suffix domain="stokke.no" /> + <suffix domain="stor-elvdal.no" /> + <suffix domain="stord.no" /> + <suffix domain="stordal.no" /> + <suffix domain="storfjord.no" /> + <suffix domain="omasvuotna.no" /> + <suffix domain="strand.no" /> + <suffix domain="stranda.no" /> + <suffix domain="stryn.no" /> + <suffix domain="sula.no" /> + <suffix domain="suldal.no" /> + <suffix domain="sund.no" /> + <suffix domain="sunndal.no" /> + <suffix domain="surnadal.no" /> + <suffix domain="sveio.no" /> + <suffix domain="svelvik.no" /> + <suffix domain="sykkylven.no" /> + <suffix domain="sogne.no" /> + <suffix domain="sÞgne.no" /> + <suffix domain="somna.no" /> + <suffix domain="sÞmna.no" /> + <suffix domain="sondre-land.no" /> + <suffix domain="sÞndre-land.no" /> + <suffix domain="sor-aurdal.no" /> + <suffix domain="sÞr-aurdal.no" /> + <suffix domain="sor-fron.no" /> + <suffix domain="sÞr-fron.no" /> + <suffix domain="sor-odal.no" /> + <suffix domain="sÞr-odal.no" /> + <suffix domain="sor-varanger.no" /> + <suffix domain="sÞr-varanger.no" /> + <suffix domain="matta-varjjat.no" /> + <suffix domain="mátta-várjjat.no" /> + <suffix domain="sorfold.no" /> + <suffix domain="sÞrfold.no" /> + <suffix domain="sorreisa.no" /> + <suffix domain="sÞrreisa.no" /> + <suffix domain="sorum.no" /> + <suffix domain="sÞrum.no" /> + <suffix domain="tana.no" /> + <suffix domain="deatnu.no" /> + <suffix domain="time.no" /> + <suffix domain="tingvoll.no" /> + <suffix domain="tinn.no" /> + <suffix domain="tjeldsund.no" /> + <suffix domain="dielddanuorri.no" /> + <suffix domain="tjome.no" /> + <suffix domain="tjÞme.no" /> + <suffix domain="tokke.no" /> + <suffix domain="tolga.no" /> + <suffix domain="torsken.no" /> + <suffix domain="tranoy.no" /> + <suffix domain="tranÞy.no" /> + <suffix domain="tromso.no" /> + <suffix domain="tromsÞ.no" /> + <suffix domain="tromsa.no" /> + <suffix domain="romsa.no" /> + <suffix domain="trondheim.no" /> + <suffix domain="troandin.no" /> + <suffix domain="trysil.no" /> + <suffix domain="trana.no" /> + <suffix domain="trÊna.no" /> + <suffix domain="trogstad.no" /> + <suffix domain="trÞgstad.no" /> + <suffix domain="tvedestrand.no" /> + <suffix domain="tydal.no" /> + <suffix domain="tynset.no" /> + <suffix domain="tysfjord.no" /> + <suffix domain="divtasvuodna.no" /> + <suffix domain="divttasvuotna.no" /> + <suffix domain="tysnes.no" /> + <suffix domain="tysvar.no" /> + <suffix domain="tysvÊr.no" /> + <suffix domain="tonsberg.no" /> + <suffix domain="tÞnsberg.no" /> + <suffix domain="ullensaker.no" /> + <suffix domain="ullensvang.no" /> + <suffix domain="ulvik.no" /> + <suffix domain="utsira.no" /> + <suffix domain="vadso.no" /> + <suffix domain="vadsÞ.no" /> + <suffix domain="cahcesuolo.no" /> + <suffix domain="cáhcesuolo.no" /> + <suffix domain="vaksdal.no" /> + <suffix domain="valle.no" /> + <suffix domain="vang.no" /> + <suffix domain="vanylven.no" /> + <suffix domain="vardo.no" /> + <suffix domain="vardÞ.no" /> + <suffix domain="varggat.no" /> + <suffix domain="várggát.no" /> + <suffix domain="vefsn.no" /> + <suffix domain="vaapste.no" /> + <suffix domain="vega.no" /> + <suffix domain="vegarshei.no" /> + <suffix domain="vegÃ¥rshei.no" /> + <suffix domain="vennesla.no" /> + <suffix domain="verdal.no" /> + <suffix domain="verran.no" /> + <suffix domain="vestby.no" /> + <suffix domain="vestnes.no" /> + <suffix domain="vestre-slidre.no" /> + <suffix domain="vestre-toten.no" /> + <suffix domain="vestvagoy.no" /> + <suffix domain="vestvÃ¥gÞy.no" /> + <suffix domain="vevelstad.no" /> + <suffix domain="vik.no" /> + <suffix domain="vikna.no" /> + <suffix domain="vindafjord.no" /> + <suffix domain="volda.no" /> + <suffix domain="voss.no" /> + <suffix domain="varoy.no" /> + <suffix domain="vÊrÞy.no" /> + <suffix domain="vagan.no" /> + <suffix domain="vÃ¥gan.no" /> + <suffix domain="voagat.no" /> + <suffix domain="vagsoy.no" /> + <suffix domain="vÃ¥gsÞy.no" /> + <suffix domain="vaga.no" /> + <suffix domain="vÃ¥gÃ¥.no" /> + <suffix domain="valer.ostfold.no" /> + <suffix domain="vÃ¥ler.Þstfold.no" /> + <suffix domain="valer.hedmark.no" /> + <suffix domain="vÃ¥ler.hedmark.no" /> + + <!-- np : http://www.mos.com.np/register.html--> + + <!-- nr : http://cenpac.net.nr/dns/index.html--> + <suffix domain="biz.nr" /> + <suffix domain="info.nr" /> + <suffix domain="gov.nr" /> + <suffix domain="edu.nr" /> + <suffix domain="org.nr" /> + <suffix domain="net.nr" /> + <suffix domain="com.nr" /> + + <!-- nu : http://en.wikipedia.org/wiki/.nu--> + + <!-- nz : http://en.wikipedia.org/wiki/.nz--> + + <!-- om : http://en.wikipedia.org/wiki/.om--> + + <!-- org : http://en.wikipedia.org/wiki/.og--> + + <!-- pa : http://www.nic.pa/--> + + <!-- pe : http://www.nic.pe/normas-proced-i.htm--> + + <!-- pf : http://www.gobin.info/domainname/formulaire-pf.pdf--> + <suffix domain="com.pf" /> + <suffix domain="org.pf" /> + <suffix domain="edu.pf" /> + + <!-- pg : http://en.wikipedia.org/wiki/.pg--> + + <!-- ph : http://www.domains.ph/FAQ2.asp--> + <!-- list of 2nd level tlds ?--> + <suffix domain="com.ph" /> + <suffix domain="net.ph" /> + <suffix domain="org.ph" /> + <suffix domain="gov.ph" /> + <suffix domain="edu.ph" /> + <suffix domain="ngo.ph" /> + <suffix domain="mil.ph" /> + + <!-- pk : http://pk5.pknic.net.pk/pk5/msgNamepk.PK--> + <suffix domain="com.pk" /> + <suffix domain="net.pk" /> + <suffix domain="edu.pk" /> + <suffix domain="org.pk" /> + <suffix domain="fam.pk" /> + <suffix domain="biz.pk" /> + <suffix domain="web.pk" /> + <suffix domain="gov.pk" /> + <suffix domain="gob.pk" /> + <suffix domain="gok.pk" /> + <suffix domain="gon.pk" /> + <suffix domain="gop.pk" /> + <suffix domain="gos.pk" /> + <suffix domain="goa.pk" /> + <suffix domain="info.pk" /> + + <!-- pl : http://www.dns.pl/english/--> + <!-- NASK functional domains (nask.pl / dns.pl) : http://www.dns.pl/english/dns-funk.html--> + <suffix domain="aid.pl" /> + <suffix domain="agro.pl" /> + <suffix domain="atm.pl" /> + <suffix domain="auto.pl" /> + <suffix domain="biz.pl" /> + <suffix domain="com.pl" /> + <suffix domain="edu.pl" /> + <suffix domain="gmina.pl" /> + <suffix domain="gsm.pl" /> + <suffix domain="info.pl" /> + <suffix domain="mail.pl" /> + <suffix domain="miasta.pl" /> + <suffix domain="media.pl" /> + <suffix domain="mil.pl" /> + <suffix domain="net.pl" /> + <suffix domain="nieruchomosci.pl" /> + <suffix domain="nom.pl" /> + <suffix domain="org.pl" /> + <suffix domain="pc.pl" /> + <suffix domain="powiat.pl" /> + <suffix domain="priv.pl" /> + <suffix domain="realestate.pl" /> + <suffix domain="rel.pl" /> + <suffix domain="sex.pl" /> + <suffix domain="shop.pl" /> + <suffix domain="sklep.pl" /> + <suffix domain="sos.pl" /> + <suffix domain="szkola.pl" /> + <suffix domain="targi.pl" /> + <suffix domain="tm.pl" /> + <suffix domain="tourism.pl" /> + <suffix domain="travel.pl" /> + <suffix domain="turystyka.pl" /> + <!-- ICM functional domains (icm.edu.pl)--> + <suffix domain="6bone.pl" /> + <suffix domain="art.pl" /> + <suffix domain="mbone.pl" /> + <!-- Government domains (administred by ippt.gov.pl)--> + <suffix domain="gov.pl" /> + <suffix domain="uw.gov.pl" /> + <suffix domain="um.gov.pl" /> + <suffix domain="ug.gov.pl" /> + <suffix domain="upow.gov.pl" /> + <suffix domain="starostwo.gov.pl" /> + <suffix domain="so.gov.pl" /> + <suffix domain="sr.gov.pl" /> + <suffix domain="po.gov.pl" /> + <suffix domain="pa.gov.pl" /> + <!-- other functional domains--> + <suffix domain="med.pl" /> + <suffix domain="ngo.pl" /> + <suffix domain="irc.pl" /> + <suffix domain="usenet.pl" /> + <!-- NASK geographical domains : http://www.dns.pl/english/dns-regiony.html--> + <suffix domain="augustow.pl" /> + <suffix domain="babia-gora.pl" /> + <suffix domain="bedzin.pl" /> + <suffix domain="beskidy.pl" /> + <suffix domain="bialowieza.pl" /> + <suffix domain="bialystok.pl" /> + <suffix domain="bielawa.pl" /> + <suffix domain="bieszczady.pl" /> + <suffix domain="boleslawiec.pl" /> + <suffix domain="bydgoszcz.pl" /> + <suffix domain="bytom.pl" /> + <suffix domain="cieszyn.pl" /> + <suffix domain="czeladz.pl" /> + <suffix domain="czest.pl" /> + <suffix domain="dlugoleka.pl" /> + <suffix domain="elblag.pl" /> + <suffix domain="elk.pl" /> + <suffix domain="glogow.pl" /> + <suffix domain="gniezno.pl" /> + <suffix domain="gorlice.pl" /> + <suffix domain="grajewo.pl" /> + <suffix domain="ilawa.pl" /> + <suffix domain="jaworzno.pl" /> + <suffix domain="jelenia-gora.pl" /> + <suffix domain="jgora.pl" /> + <suffix domain="kalisz.pl" /> + <suffix domain="kazimierz-dolny.pl" /> + <suffix domain="karpacz.pl" /> + <suffix domain="kartuzy.pl" /> + <suffix domain="kaszuby.pl" /> + <suffix domain="katowice.pl" /> + <suffix domain="kepno.pl" /> + <suffix domain="ketrzyn.pl" /> + <suffix domain="klodzko.pl" /> + <suffix domain="kobierzyce.pl" /> + <suffix domain="kolobrzeg.pl" /> + <suffix domain="konin.pl" /> + <suffix domain="konskowola.pl" /> + <suffix domain="kutno.pl" /> + <suffix domain="lapy.pl" /> + <suffix domain="lebork.pl" /> + <suffix domain="legnica.pl" /> + <suffix domain="lezajsk.pl" /> + <suffix domain="limanowa.pl" /> + <suffix domain="lomza.pl" /> + <suffix domain="lowicz.pl" /> + <suffix domain="lubin.pl" /> + <suffix domain="lukow.pl" /> + <suffix domain="malbork.pl" /> + <suffix domain="malopolska.pl" /> + <suffix domain="mazowsze.pl" /> + <suffix domain="mazury.pl" /> + <suffix domain="mielec.pl" /> + <suffix domain="mielno.pl" /> + <suffix domain="mragowo.pl" /> + <suffix domain="naklo.pl" /> + <suffix domain="nowaruda.pl" /> + <suffix domain="nysa.pl" /> + <suffix domain="olawa.pl" /> + <suffix domain="olecko.pl" /> + <suffix domain="olkusz.pl" /> + <suffix domain="olsztyn.pl" /> + <suffix domain="opoczno.pl" /> + <suffix domain="opole.pl" /> + <suffix domain="ostroda.pl" /> + <suffix domain="ostroleka.pl" /> + <suffix domain="ostrowiec.pl" /> + <suffix domain="ostrowwlkp.pl" /> + <suffix domain="pila.pl" /> + <suffix domain="pisz.pl" /> + <suffix domain="podhale.pl" /> + <suffix domain="podlasie.pl" /> + <suffix domain="polkowice.pl" /> + <suffix domain="pomorze.pl" /> + <suffix domain="pomorskie.pl" /> + <suffix domain="prochowice.pl" /> + <suffix domain="pruszkow.pl" /> + <suffix domain="przeworsk.pl" /> + <suffix domain="pulawy.pl" /> + <suffix domain="radom.pl" /> + <suffix domain="rawa-maz.pl" /> + <suffix domain="rybnik.pl" /> + <suffix domain="rzeszow.pl" /> + <suffix domain="sanok.pl" /> + <suffix domain="sejny.pl" /> + <suffix domain="slask.pl" /> + <suffix domain="slupsk.pl" /> + <suffix domain="sosnowiec.pl" /> + <suffix domain="stalowa-wola.pl" /> + <suffix domain="skoczow.pl" /> + <suffix domain="starachowice.pl" /> + <suffix domain="stargard.pl" /> + <suffix domain="suwalki.pl" /> + <suffix domain="swidnica.pl" /> + <suffix domain="swiebodzin.pl" /> + <suffix domain="swinoujscie.pl" /> + <suffix domain="szczecin.pl" /> + <suffix domain="szczytno.pl" /> + <suffix domain="tarnobrzeg.pl" /> + <suffix domain="tgory.pl" /> + <suffix domain="turek.pl" /> + <suffix domain="tychy.pl" /> + <suffix domain="ustka.pl" /> + <suffix domain="walbrzych.pl" /> + <suffix domain="warmia.pl" /> + <suffix domain="warszawa.pl" /> + <suffix domain="waw.pl" /> + <suffix domain="wegrow.pl" /> + <suffix domain="wielun.pl" /> + <suffix domain="wlocl.pl" /> + <suffix domain="wloclawek.pl" /> + <suffix domain="wodzislaw.pl" /> + <suffix domain="wolomin.pl" /> + <suffix domain="wroclaw.pl" /> + <suffix domain="zachpomor.pl" /> + <suffix domain="zagan.pl" /> + <suffix domain="zarow.pl" /> + <suffix domain="zgora.pl" /> + <suffix domain="zgorzelec.pl" /> + <!-- TASK geographical domains (www.task.gda.pl/uslugi/dns)--> + <suffix domain="gda.pl" /> + <suffix domain="gdansk.pl" /> + <suffix domain="gdynia.pl" /> + <suffix domain="sopot.pl" /> + <!-- other geographical domains--> + <suffix domain="gliwice.pl" /> + <suffix domain="krakow.pl" /> + <suffix domain="poznan.pl" /> + <suffix domain="wroc.pl" /> + <suffix domain="zakopane.pl" /> + + <!-- pn : http://www.government.pn/PnRegistry/policies.htm--> + <suffix domain="gov.pn" /> + <suffix domain="co.pn" /> + <suffix domain="org.pn" /> + <suffix domain="edu.pn" /> + <suffix domain="net.pn" /> + + <!-- pr : http://www.nic.pr/index.asp?f=1--> + <suffix domain="com.pr" /> + <suffix domain="net.pr" /> + <suffix domain="org.pr" /> + <suffix domain="gov.pr" /> + <suffix domain="edu.pr" /> + <suffix domain="isla.pr" /> + <suffix domain="pro.pr" /> + <suffix domain="biz.pr" /> + <suffix domain="info.pr" /> + <suffix domain="name.pr" /> + <!-- these aren't mentioned on nic.pr, but on http://en.wikipedia.org/wiki/.pr--> + <suffix domain="est.pr" /> + <suffix domain="prof.pr" /> + <suffix domain="ac.pr" /> + + <!-- pro : http://www.nic.pro/support_faq.htm--> + <suffix domain="aca.pro" /> + <suffix domain="bar.pro" /> + <suffix domain="cpa.pro" /> + <suffix domain="jur.pro" /> + <suffix domain="law.pro" /> + <suffix domain="med.pro" /> + <suffix domain="eng.pro" /> + + <!-- ps : http://en.wikipedia.org/wiki/.ps--> + <!-- list of 2nd level tlds ?--> + <suffix domain="edu.ps" /> + <suffix domain="gov.ps" /> + <suffix domain="sec.ps" /> + <suffix domain="plo.ps" /> + <suffix domain="com.ps" /> + <suffix domain="org.ps" /> + <suffix domain="net.ps" /> + + <!-- pt : http://online.dns.pt/dns/start_dns--> + <suffix domain="net.pt" /> + <suffix domain="gov.pt" /> + <suffix domain="org.pt" /> + <suffix domain="edu.pt" /> + <suffix domain="int.pt" /> + <suffix domain="publ.pt" /> + <suffix domain="com.pt" /> + <suffix domain="nome.pt" /> + + <!-- pw : http://en.wikipedia.org/wiki/.pw--> + + <!-- py : http://www.nic.py/faq_a.html#faq_b--> + + <!-- qa : http://www.qatar.net.qa/services/virtual.htm--> + + <!-- re : http://www.afnic.re/obtenir/chartes/nommage-re/annexe-descriptifs--> + <suffix domain="com.re" /> + <suffix domain="asso.re" /> + <suffix domain="nom.re" /> + + <!-- ro : http://www.rotld.ro/--> + <suffix domain="com.ro" /> + <suffix domain="org.ro" /> + <suffix domain="tm.ro" /> + <suffix domain="nt.ro" /> + <suffix domain="nom.ro" /> + <suffix domain="info.ro" /> + <suffix domain="rec.ro" /> + <suffix domain="arts.ro" /> + <suffix domain="firm.ro" /> + <suffix domain="store.ro" /> + <suffix domain="www.ro" /> + + <!-- ru : http://en.wikipedia.org/wiki/.ru--> + <suffix domain="com.ru" /> + <suffix domain="net.ru" /> + <suffix domain="org.ru" /> + <suffix domain="pp.ru" /> + <suffix domain="int.ru" /> + <!-- there should be geo-names like msk.ru, but I didn't find a list--> + + <!-- rw : http://www.nic.rw/cgi-bin/policy.pl--> + <suffix domain="gov.rw" /> + <suffix domain="net.rw" /> + <suffix domain="edu.rw" /> + <suffix domain="ac.rw" /> + <suffix domain="com.rw" /> + <suffix domain="co.rw" /> + <suffix domain="int.rw" /> + <suffix domain="mil.rw" /> + <suffix domain="gouv.rw" /> + + <!-- sa : http://www.saudinic.net.sa/page.php?page=1&lang=1--> + + <!-- sb : http://www.sbnic.net.sb/--> + + <!-- sc : http://www.nic.sc/--> + <suffix domain="com.sc" /> + <suffix domain="gov.sc" /> + <suffix domain="net.sc" /> + <suffix domain="org.sc" /> + <suffix domain="edu.sc" /> + + <!-- sd : http://www.isoc.sd/sudanic.isoc.sd/billing_pricing.htm--> + <suffix domain="com.sd" /> + <suffix domain="net.sd" /> + <suffix domain="org.sd" /> + <suffix domain="edu.sd" /> + <suffix domain="med.sd" /> + <suffix domain="tv.sd" /> + <suffix domain="gov.sd" /> + <suffix domain="info.sd" /> + + <!-- se : http://en.wikipedia.org/wiki/.se--> + <suffix domain="org.se" /> + <suffix domain="pp.se" /> + <suffix domain="tm.se" /> + <suffix domain="parti.se" /> + <suffix domain="press.se" /> + <suffix domain="mil.se" /> + <!-- se geographical names--> + <suffix domain="ab.se" /> + <suffix domain="c.se" /> + <suffix domain="d.se" /> + <suffix domain="e.se" /> + <suffix domain="f.se" /> + <suffix domain="g.se" /> + <suffix domain="h.se" /> + <suffix domain="i.se" /> + <suffix domain="k.se" /> + <suffix domain="m.se" /> + <suffix domain="n.se" /> + <suffix domain="o.se" /> + <suffix domain="s.se" /> + <suffix domain="t.se" /> + <suffix domain="u.se" /> + <suffix domain="w.se" /> + <suffix domain="x.se" /> + <suffix domain="y.se" /> + <suffix domain="z.se" /> + <suffix domain="ac.se" /> + <suffix domain="bd.se" /> + + <!-- sg : http://www.nic.net.sg/sub_policies_agreement/2ld.html--> + <suffix domain="com.sg" /> + <suffix domain="net.sg" /> + <suffix domain="org.sg" /> + <suffix domain="gov.sg" /> + <suffix domain="edu.sg" /> + <suffix domain="per.sg" /> + + <!-- sh : http://www.nic.sh/rules.html--> + <!-- list of 2nd level domains ?--> + + <!-- si : http://en.wikipedia.org/wiki/.si--> + + <!-- sk : http://en.wikipedia.org/wiki/.sk--> + + <!-- sl : http://en.wikipedia.org/wiki/.sl--> + <!-- list of 2nd level domains ?--> + + <!-- sm : http://en.wikipedia.org/wiki/.sm--> + + <!-- sn : http://en.wikipedia.org/wiki/.sn--> + <!-- list of 2nd level domains ?--> + + <!-- sr : http://en.wikipedia.org/wiki/.sr--> + + <!-- st : http://www.nic.st/html/policyrules/--> + + <!-- su : http://en.wikipedia.org/wiki/.su--> + + <!-- sv : http://www.svnet.org.sv/svpolicy.html--> + + <!-- sy : http://www.gobin.info/domainname/sy.doc--> + + <!-- sz : http://en.wikipedia.org/wiki/.sz--> + <!-- list of 2nd level domains ?--> + + <!-- tc : http://en.wikipedia.org/wiki/.tc--> + + <!-- td : http://en.wikipedia.org/wiki/.td--> + + <!-- tf : http://en.wikipedia.org/wiki/.tf--> + + <!-- tg : http://en.wikipedia.org/wiki/.tg--> + <!-- list of 2nd level domains ?--> + + <!-- th : http://en.wikipedia.org/wiki/.th--> + + <!-- tj : http://www.nic.tj/policy.htm--> + <suffix domain="ac.tj" /> + <suffix domain="biz.tj" /> + <suffix domain="com.tj" /> + <suffix domain="co.tj" /> + <suffix domain="edu.tj" /> + <suffix domain="int.tj" /> + <suffix domain="name.tj" /> + <suffix domain="net.tj" /> + <suffix domain="org.tj" /> + <suffix domain="web.tj" /> + <suffix domain="gov.tj" /> + <suffix domain="go.tj" /> + <suffix domain="mil.tj" /> + + <!-- tk : http://en.wikipedia.org/wiki/.tk--> + + <!-- tl : http://en.wikipedia.org/wiki/.tl--> + <!-- list of 2nd level tlds ?--> + + <!-- tm : http://www.nic.tm/rules.html--> + <!-- list of 2nd level tlds ?--> + + <!-- tn : http://en.wikipedia.org/wiki/.tn--> + <!-- list of 2nd level tlds ?--> + + <!-- to : http://en.wikipedia.org/wiki/.to--> + <!-- list of 2nd level tlds ?--> + + <!-- tr : http://en.wikipedia.org/wiki/.tr--> + + <!-- travel : http://en.wikipedia.org/wiki/.travel--> + + <!-- tt : http://www.nic.tt/--> + <suffix domain="co.tt" /> + <suffix domain="com.tt" /> + <suffix domain="org.tt" /> + <suffix domain="net.tt" /> + <suffix domain="biz.tt" /> + <suffix domain="info.tt" /> + <suffix domain="pro.tt" /> + <suffix domain="int.tt" /> + <suffix domain="coop.tt" /> + <suffix domain="jobs.tt" /> + <suffix domain="mobi.tt" /> + <suffix domain="travel.tt" /> + <suffix domain="museum.tt" /> + <suffix domain="aero.tt" /> + <suffix domain="name.tt" /> + <suffix domain="gov.tt" /> + <suffix domain="edu.tt" /> + + <!-- tv : http://en.wikipedia.org/wiki/.tv--> + <!-- list of 2nd level tlds ?--> + + <!-- tw : http://en.wikipedia.org/wiki/.tw--> + <suffix domain="edu.tw" /> + <suffix domain="gov.tw" /> + <suffix domain="mil.tw" /> + <suffix domain="com.tw" /> + <suffix domain="net.tw" /> + <suffix domain="org.tw" /> + <suffix domain="idv.tw" /> + <suffix domain="game.tw" /> + <suffix domain="ebiz.tw" /> + <suffix domain="club.tw" /> + <suffix domain="網路.tw" /> + <suffix domain="組織.tw" /> + <suffix domain="商æ¥Â.tw" /> + + <!-- tz : http://en.wikipedia.org/wiki/.tz--> + + <!-- ua : http://www.nic.net.ua/--> + <suffix domain="com.ua" /> + <suffix domain="edu.ua" /> + <suffix domain="gov.ua" /> + <suffix domain="net.ua" /> + <suffix domain="org.ua" /> + <!-- ua geo-names--> + <suffix domain="cherkassy.ua" /> + <suffix domain="chernigov.ua" /> + <suffix domain="chernovtsy.ua" /> + <suffix domain="ck.ua" /> + <suffix domain="cn.ua" /> + <suffix domain="crimea.ua" /> + <suffix domain="cv.ua" /> + <suffix domain="dn.ua" /> + <suffix domain="dnepropetrovsk.ua" /> + <suffix domain="donetsk.ua" /> + <suffix domain="dp.ua" /> + <suffix domain="if.ua" /> + <suffix domain="ivano-frankivsk.ua" /> + <suffix domain="kh.ua" /> + <suffix domain="kharkov.ua" /> + <suffix domain="kherson.ua" /> + <suffix domain="kiev.ua" /> + <suffix domain="kirovograd.ua" /> + <suffix domain="km.ua" /> + <suffix domain="kr.ua" /> + <suffix domain="ks.ua" /> + <suffix domain="lg.ua" /> + <suffix domain="lugansk.ua" /> + <suffix domain="lutsk.ua" /> + <suffix domain="lviv.ua" /> + <suffix domain="mk.ua" /> + <suffix domain="nikolaev.ua" /> + <suffix domain="od.ua" /> + <suffix domain="odessa.ua" /> + <suffix domain="pl.ua" /> + <suffix domain="poltava.ua" /> + <suffix domain="rovno.ua" /> + <suffix domain="rv.ua" /> + <suffix domain="sebastopol.ua" /> + <suffix domain="sumy.ua" /> + <suffix domain="te.ua" /> + <suffix domain="ternopil.ua" /> + <suffix domain="vinnica.ua" /> + <suffix domain="vn.ua" /> + <suffix domain="zaporizhzhe.ua" /> + <suffix domain="zp.ua" /> + <suffix domain="uz.ua" /> + <suffix domain="uzhgorod.ua" /> + <suffix domain="zhitomir.ua" /> + <suffix domain="zt.ua" /> + + <!-- ug : http://www.registry.co.ug/--> + <suffix domain="co.ug" /> + <suffix domain="ac.ug" /> + <suffix domain="sc.ug" /> + <suffix domain="go.ug" /> + <suffix domain="ne.ug" /> + <suffix domain="or.ug" /> + + <!-- uk : http://en.wikipedia.org/wiki/.uk--> + <suffix domain="sch.uk" /> + <suffix domain="bl.uk" /> + <suffix domain="british-library.uk" /> + <suffix domain="icnet.uk" /> + <suffix domain="jet.uk" /> + <suffix domain="nel.uk" /> + <suffix domain="nls.uk" /> + <suffix domain="national-library-scotland.uk" /> + <suffix domain="parliament.uk" /> + + <!-- us : http://en.wikipedia.org/wiki/.us--> + <suffix domain="dni.us" /> + <suffix domain="fed.us" /> + <suffix domain="isa.us" /> + <suffix domain="kids.us" /> + <suffix domain="nsn.us" /> + <!-- us geographic names--> + <suffix domain="ak.us" /> + <suffix domain="al.us" /> + <suffix domain="ar.us" /> + <suffix domain="az.us" /> + <suffix domain="ca.us" /> + <suffix domain="co.us" /> + <suffix domain="ct.us" /> + <suffix domain="dc.us" /> + <suffix domain="de.us" /> + <suffix domain="fl.us" /> + <suffix domain="ga.us" /> + <suffix domain="hi.us" /> + <suffix domain="ia.us" /> + <suffix domain="id.us" /> + <suffix domain="il.us" /> + <suffix domain="in.us" /> + <suffix domain="ks.us" /> + <suffix domain="ky.us" /> + <suffix domain="la.us" /> + <suffix domain="ma.us" /> + <suffix domain="md.us" /> + <suffix domain="me.us" /> + <suffix domain="mi.us" /> + <suffix domain="mn.us" /> + <suffix domain="mo.us" /> + <suffix domain="ms.us" /> + <suffix domain="mt.us" /> + <suffix domain="nc.us" /> + <suffix domain="nd.us" /> + <suffix domain="ne.us" /> + <suffix domain="nh.us" /> + <suffix domain="nj.us" /> + <suffix domain="nm.us" /> + <suffix domain="nv.us" /> + <suffix domain="ny.us" /> + <suffix domain="oh.us" /> + <suffix domain="ok.us" /> + <suffix domain="or.us" /> + <suffix domain="pa.us" /> + <suffix domain="ri.us" /> + <suffix domain="sc.us" /> + <suffix domain="sd.us" /> + <suffix domain="tn.us" /> + <suffix domain="tx.us" /> + <suffix domain="ut.us" /> + <suffix domain="vt.us" /> + <suffix domain="va.us" /> + <suffix domain="wa.us" /> + <suffix domain="wi.us" /> + <suffix domain="wv.us" /> + <suffix domain="wy.us" /> + <!-- the following rules would be only valid under the geo-name, but we can't express that--> + <!-- *.*.us cities, counties, parishes, and townships (locality.state.us)--> + <!-- !ci.*.*.us city government agencies (subdomain under locality)--> + <!-- !town.*.*.us town government agencies (subdomain under locality)--> + <!-- !co.*.*.us county government agencies (subdomain under locality)--> + <!-- k12.*.us public school districts--> + <!-- pvt.k12.*.us private schools--> + <!-- cc.*.us community colleges--> + <!-- tec.*.us technical and vocational schools--> + <!-- lib.*.us state, regional, city, and county libraries--> + <!-- state.*.us state government agencies--> + <!-- gen.*.us general independent entities (groups not fitting into the above categories)--> + + <!-- uy : http://www.antel.com.uy/--> + + <!-- uz : http://www.reg.uz/registerr.html--> + <!-- are there other 2nd level tlds ?--> + <suffix domain="com.uz" /> + <suffix domain="co.uz" /> + + <!-- va : http://en.wikipedia.org/wiki/.va--> + + <!-- vc : http://en.wikipedia.org/wiki/.vc--> + <!-- list of 2nd level tlds ?--> + + <!-- ve : http://registro.nic.ve/nicve/registro/index.html--> + + <!-- vg : http://en.wikipedia.org/wiki/.vg--> + + <!-- vi : http://www.nic.vi/Domain_Rules/body_domain_rules.html--> + <suffix domain="com.vi" /> + <suffix domain="org.vi" /> + <suffix domain="edu.vi" /> + <suffix domain="gov.vi" /> + + <!-- vn : https://www.dot.vn/vnnic/vnnic/domainregistration.jsp--> + <suffix domain="com.vn" /> + <suffix domain="net.vn" /> + <suffix domain="org.vn" /> + <suffix domain="edu.vn" /> + <suffix domain="gov.vn" /> + <suffix domain="int.vn" /> + <suffix domain="ac.vn" /> + <suffix domain="biz.vn" /> + <suffix domain="info.vn" /> + <suffix domain="name.vn" /> + <suffix domain="pro.vn" /> + <suffix domain="health.vn" /> + + <!-- vu : http://en.wikipedia.org/wiki/.vu--> + <!-- list of 2nd level tlds ?--> + + <!-- ws : http://en.wikipedia.org/wiki/.ws--> + + <!-- ye : http://www.y.net.ye/services/domain_name.htm--> + + <!-- yu : http://www.nic.yu/pravilnik-e.html--> + + <!-- za : http://www.zadna.org.za/slds.html--> + + <!-- zm : http://en.wikipedia.org/wiki/.zm--> + + <!-- zw : http://en.wikipedia.org/wiki/.zw--> + + </suffixes> +</domains> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xsd b/search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xsd new file mode 100644 index 0000000..ba68d62 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/domain-suffixes.xsd @@ -0,0 +1,130 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Document : domain-suffixes.xsd + Author : Enis Soztutar - enis.soz.nutch@gmail.com + Description: This document is the schema for valid domain-suffixes + definitions. For successful parsing of domain-suffixes xml files, + the xml file should be validated with this xsd. + See : org.apache.nutch.util.domain.DomainSuffixesReader.java +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" + targetNamespace="http://lucene.apache.org/nutch" + xmlns="http://lucene.apache.org/nutch" + elementFormDefault="qualified"> + + <xs:element name="domains"> + xs:complexType + xs:sequence + <xs:element name="tlds"> + xs:complexType + xs:sequence + <xs:element name="itlds"> + xs:complexType + xs:sequence + <xs:element name="tld" maxOccurs="unbounded" + type="gtld" /> + </xs:sequence> + </xs:complexType> + </xs:element> + + <xs:element name="gtlds"> + xs:complexType + xs:sequence + <xs:element name="tld" maxOccurs="unbounded" + type="gtld" /> + </xs:sequence> + </xs:complexType> + </xs:element> + + <xs:element name="cctlds"> + xs:complexType + xs:sequence + <xs:element name="tld" maxOccurs="unbounded" + type="cctld" /> + </xs:sequence> + </xs:complexType> + </xs:element> + + </xs:sequence> + </xs:complexType> + </xs:element> + + <xs:element name="suffixes"> + xs:complexType + xs:sequence + <xs:element name="suffix" maxOccurs="unbounded" + type="sldType" /> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + + <xs:complexType name="gtld"> + xs:sequence + <xs:element name="status" minOccurs="0"> + xs:simpleType + <xs:restriction base="xs:string"> + <xs:enumeration value="INFRASTRUCTURE" /> + <xs:enumeration value="SPONSORED" /> + <xs:enumeration value="UNSPONSORED" /> + <xs:enumeration value="STARTUP" /> + <xs:enumeration value="PROPOSED" /> + <xs:enumeration value="DELETED" /> + <xs:enumeration value="PSEUDO_DOMAIN" /> + </xs:restriction> + </xs:simpleType> + </xs:element> + <xs:element name="boost" type="xs:float" minOccurs="0" /> + <xs:element name="description" type="xs:string" minOccurs="0" /> + </xs:sequence> + <xs:attribute name="domain" type="xs:string" /> + </xs:complexType> + + <xs:complexType name="cctld"> + xs:sequence + <xs:element name="country" type="xs:string" /> + <xs:element name="status" type="statusType" minOccurs="0" /> + <xs:element name="boost" type="xs:float" minOccurs="0" /> + <xs:element name="description" type="xs:string" minOccurs="0" /> + </xs:sequence> + <xs:attribute name="domain" type="xs:string" /> + </xs:complexType> + + <xs:complexType name="sldType"> + xs:sequence + <xs:element name="status" type="statusType" minOccurs="0" /> + <xs:element name="boost" type="xs:float" minOccurs="0" /> + <xs:element name="description" type="xs:string" minOccurs="0" /> + </xs:sequence> + <xs:attribute name="domain" type="xs:string" /> + </xs:complexType> + + <xs:simpleType name="statusType"> + <xs:restriction base="xs:string"> + <xs:enumeration value="IN_USE" /> + <xs:enumeration value="NOT_IN_USE" /> + <xs:enumeration value="DELETED" /> + </xs:restriction> + </xs:simpleType> + +</xs:schema> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-env.sh b/search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-env.sh new file mode 100644 index 0000000..9050a6e --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-env.sh @@ -0,0 +1,33 @@ +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. Required. +# export JAVA_HOME=/usr/bin/java + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HADOOP_HEAPSIZE=2000 + +# Extra Java runtime options. Empty by default. +# export HADOOP_OPTS=-server + +# Extra ssh options. Default: '-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR'. +# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR" + +# Where log files are stored. $HADOOP_HOME/logs by default. +# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs + +# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. +# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves + +# host:path where hadoop code should be rsync'd from. Unset by default. +# export HADOOP_MASTER=master:/home/$USER/src/hadoop + +# The directory where pid files are stored. /tmp by default. +# export HADOOP_PID_DIR=/var/hadoop/pids + +# A string representing this instance of hadoop. $USER by default. +# export HADOOP_IDENT_STRING=$USER diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-site.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-site.xml new file mode 100644 index 0000000..970c8fe --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/hadoop-site.xml @@ -0,0 +1,8 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + +</configuration> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/httpclient-auth.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/httpclient-auth.xml new file mode 100644 index 0000000..60d45b9 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/httpclient-auth.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<!-- + This is the authentication configuration file for protocol-httpclient. + Different credentials for different authentication scopes can be + configured in this file. If a set of credentials is configured for a + particular authentication scope (i.e. particular host, port number, + scheme and realm), then that set of credentials would be sent only to + servers falling under the specified authentication scope. Apart from + this at most one set of credentials can be configured as 'default'. + + When authentication is required to fetch a resource from a web-server, + the authentication-scope is determined from the host, port, scheme and + realm (if present) obtained from the URL of the page and the + authentication headers in the HTTP response. If it matches any + 'authscope' in this configuration file, then the 'credentials' for + that 'authscope' is used for authentication. Otherwise, it would use + the 'default' set of credentials (with an exception which is described + in the next paragraph), if present. If any attribute is missing, it + would match all values for that attribute. + + If there are several pages having different authentication realms and + schemes on the same web-server (same host and port, but different + realms and schemes), and credentials for one or more of the realms and + schemes for that web-server is specified, then the 'default' + credentials would be ignored completely for that web-server (for that + host and port). So, credentials to handle all realms and schemes for + that server may be specified explicitly by adding an extra 'authscope' + tag with the 'realm' and 'scheme' attributes missing for that server. + This is demonstrated by the last 'authscope' tag for 'example:8080' in + the following example. + + Example:- + <credentials username="susam" password="masus"> + <default realm="sso"/> + <authscope host="192.168.101.33" port="80" realm="login"/> + <authscope host="example" port="8080" realm="blogs"/> + <authscope host="example" port="8080" realm="wiki"/> + <authscope host="example" port="80" realm="quiz" scheme="NTLM"/> + </credentials> + <credentials username="admin" password="nimda"> + <authscope host="example" port="8080"/> + </credentials> + + In the above example, 'example:8080' server has pages with multiple + authentication realms. The first set of credentials would be used for + 'blogs' and 'wiki' authentication realms. The second set of + credentials would be used for all other realms. For 'login' realm of + '192.168.101.33', the first set of credentials would be used. For any + other realm of '192.168.101.33' authentication would not be done. For + the NTLM authentication required by 'example:80', the first set of + credentials would be used. For 'sso' realms of all other servers, the + first set of credentials would be used, since it is configured as + 'default'. + + NTLM does not use the notion of realms. The domain name may be + specified as the value for 'realm' attribute in case of NTLM. +--> + +<auth-configuration> + +</auth-configuration> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/log4j.properties b/search-server/spacewalk-search/nutch/crawl_jsp/conf/log4j.properties new file mode 100644 index 0000000..5a985ab --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/log4j.properties @@ -0,0 +1,82 @@ +# Define some default values that can be overridden by system properties +hadoop.log.dir=. +hadoop.log.file=hadoop.log + +# RootLogger - DailyRollingFileAppender +log4j.rootLogger=ALL,DRFA + +# Logging Threshold +log4j.threshhold=ALL + +#special logging requirements for some commandline tools +log4j.logger.org.apache.nutch.crawl.Crawl=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.Injector=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.Generator=INFO,cmdstdout +log4j.logger.org.apache.nutch.fetcher.Fetcher=INFO,cmdstdout +log4j.logger.org.apache.nutch.parse.ParseSegment=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.CrawlDbReader=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.CrawlDbMerger=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.LinkDbReader=INFO,cmdstdout +log4j.logger.org.apache.nutch.segment.SegmentReader=INFO,cmdstdout +log4j.logger.org.apache.nutch.segment.SegmentMerger=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.CrawlDb=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.LinkDb=INFO,cmdstdout +log4j.logger.org.apache.nutch.crawl.LinkDbMerger=INFO,cmdstdout +log4j.logger.org.apache.nutch.indexer.Indexer=INFO,cmdstdout +log4j.logger.org.apache.nutch.indexer.DeleteDuplicates=INFO,cmdstdout +log4j.logger.org.apache.nutch.indexer.IndexMerger=INFO,cmdstdout + +log4j.logger.org.apache.nutch=INFO +log4j.logger.org.apache.hadoop=WARN + +# +# Daily Rolling File Appender +# + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n +# Debugging Pattern format: Date LogLevel LoggerName (FileName:MethodName:LineNo) LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# stdout +# Add *stdout* to rootlogger above if you want to use this +# + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# +# plain layout used for commandline tools to output to console +# +log4j.appender.cmdstdout=org.apache.log4j.ConsoleAppender +log4j.appender.cmdstdout.layout=org.apache.log4j.PatternLayout +log4j.appender.cmdstdout.layout.ConversionPattern=%m%n + +# +# Rolling File Appender +# + +#log4j.appender.RFA=org.apache.log4j.RollingFileAppender +#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Logfile size and and 30-day backups +#log4j.appender.RFA.MaxFileSize=1MB +#log4j.appender.RFA.MaxBackupIndex=30 + +#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n +#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/logging.properties b/search-server/spacewalk-search/nutch/crawl_jsp/conf/logging.properties new file mode 100644 index 0000000..67b4a73 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/logging.properties @@ -0,0 +1,4 @@ +handlers= java.util.logging.ConsoleHandler +.level= FINE + + diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/masters b/search-server/spacewalk-search/nutch/crawl_jsp/conf/masters new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/masters @@ -0,0 +1 @@ +localhost diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-conf.xsl b/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-conf.xsl new file mode 100644 index 0000000..36a2275 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-conf.xsl @@ -0,0 +1,24 @@ +<?xml version="1.0"?> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> +<xsl:output method="html"/> +<xsl:template match="nutch-conf"> +<html> +<body> +<table border="1"> +<tr> + <td>name</td> + <td>value</td> + <td>description</td> +</tr> +<xsl:for-each select="property"> +<tr> + <td><xsl:value-of select="name"/></td> + <td><xsl:value-of select="value"/></td> + <td><xsl:value-of select="description"/></td> +</tr> +</xsl:for-each> +</table> +</body> +</html> +</xsl:template> +</xsl:stylesheet> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-default.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-default.xml new file mode 100644 index 0000000..2bf0e1e --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-default.xml @@ -0,0 +1,1205 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- Do not modify this file directly. Instead, copy entries that you --> +<!-- wish to modify from this file into nutch-site.xml and change them --> +<!-- there. If nutch-site.xml does not already exist, create it. --> + +<configuration> + +<!-- file properties --> + +<property> + <name>file.content.limit</name> + <value>65536</value> + <description>The length limit for downloaded content, in bytes. + If this value is nonnegative (>=0), content longer than it will be truncated; + otherwise, no truncation at all. + </description> +</property> + +<property> + <name>file.content.ignored</name> + <value>true</value> + <description>If true, no file content will be saved during fetch. + And it is probably what we want to set most of time, since file:// URLs + are meant to be local and we can always use them directly at parsing + and indexing stages. Otherwise file contents will be saved. + !! NO IMPLEMENTED YET !! + </description> +</property> + +<!-- HTTP properties --> + +<property> + <name>http.agent.name</name> + <value></value> + <description>HTTP 'User-Agent' request header. MUST NOT be empty - + please set this to a single word uniquely related to your organization. + + NOTE: You should also check other related properties: + + http.robots.agents + http.agent.description + http.agent.url + http.agent.email + http.agent.version + + and set their values appropriately. + + </description> +</property> + +<property> + <name>http.robots.agents</name> + <value>*</value> + <description>The agent strings we'll look for in robots.txt files, + comma-separated, in decreasing order of precedence. You should + put the value of http.agent.name as the first agent name, and keep the + default * at the end of the list. E.g.: BlurflDev,Blurfl,* + </description> +</property> + +<property> + <name>http.robots.403.allow</name> + <value>true</value> + <description>Some servers return HTTP status 403 (Forbidden) if + /robots.txt doesn't exist. This should probably mean that we are + allowed to crawl the site nonetheless. If this is set to false, + then such sites will be treated as forbidden.</description> +</property> + +<property> + <name>http.agent.description</name> + <value></value> + <description>Further description of our bot- this text is used in + the User-Agent header. It appears in parenthesis after the agent name. + </description> +</property> + +<property> + <name>http.agent.url</name> + <value></value> + <description>A URL to advertise in the User-Agent header. This will + appear in parenthesis after the agent name. Custom dictates that this + should be a URL of a page explaining the purpose and behavior of this + crawler. + </description> +</property> + +<property> + <name>http.agent.email</name> + <value></value> + <description>An email address to advertise in the HTTP 'From' request + header and User-Agent header. A good practice is to mangle this + address (e.g. 'info at example dot com') to avoid spamming. + </description> +</property> + +<property> + <name>http.agent.version</name> + <value>Nutch-1.0-dev</value> + <description>A version string to advertise in the User-Agent + header.</description> +</property> + +<property> + <name>http.agent.host</name> + <value></value> + <description>Name or IP address of the host on which the Nutch crawler + would be running. Currently this is used by 'protocol-httpclient' + plugin. + </description> +</property> + +<property> + <name>http.timeout</name> + <value>10000</value> + <description>The default network timeout, in milliseconds.</description> +</property> + +<property> + <name>http.max.delays</name> + <value>100</value> + <description>The number of times a thread will delay when trying to + fetch a page. Each time it finds that a host is busy, it will wait + fetcher.server.delay. After http.max.delays attepts, it will give + up on the page for now.</description> +</property> + +<property> + <name>http.content.limit</name> + <value>65536</value> + <description>The length limit for downloaded content, in bytes. + If this value is nonnegative (>=0), content longer than it will be truncated; + otherwise, no truncation at all. + </description> +</property> + +<property> + <name>http.proxy.host</name> + <value></value> + <description>The proxy hostname. If empty, no proxy is used.</description> +</property> + +<property> + <name>http.proxy.port</name> + <value></value> + <description>The proxy port.</description> +</property> + +<property> + <name>http.proxy.username</name> + <value></value> + <description>Username for proxy. This will be used by + 'protocol-httpclient', if the proxy server requests basic, digest + and/or NTLM authentication. To use this, 'protocol-httpclient' must + be present in the value of 'plugin.includes' property. + NOTE: For NTLM authentication, do not prefix the username with the + domain, i.e. 'susam' is correct whereas 'DOMAIN\susam' is incorrect. + </description> +</property> + +<property> + <name>http.proxy.password</name> + <value></value> + <description>Password for proxy. This will be used by + 'protocol-httpclient', if the proxy server requests basic, digest + and/or NTLM authentication. To use this, 'protocol-httpclient' must + be present in the value of 'plugin.includes' property. + </description> +</property> + +<property> + <name>http.proxy.realm</name> + <value></value> + <description>Authentication realm for proxy. Do not define a value + if realm is not required or authentication should take place for any + realm. NTLM does not use the notion of realms. Specify the domain name + of NTLM authentication as the value for this property. To use this, + 'protocol-httpclient' must be present in the value of + 'plugin.includes' property. + </description> +</property> + +<property> + <name>http.auth.file</name> + <value>httpclient-auth.xml</value> + <description>Authentication configuration file for + 'protocol-httpclient' plugin. + </description> +</property> + +<property> + <name>http.verbose</name> + <value>false</value> + <description>If true, HTTP will log more verbosely.</description> +</property> + +<property> + <name>http.redirect.max</name> + <value>0</value> + <description>The maximum number of redirects the fetcher will follow when + trying to fetch a page. If set to negative or 0, fetcher won't immediately + follow redirected URLs, instead it will record them for later fetching. + </description> +</property> + +<property> + <name>http.useHttp11</name> + <value>false</value> + <description>NOTE: at the moment this works only for protocol-httpclient. + If true, use HTTP 1.1, if false use HTTP 1.0 . + </description> +</property> + +<!-- FTP properties --> + +<property> + <name>ftp.username</name> + <value>anonymous</value> + <description>ftp login username.</description> +</property> + +<property> + <name>ftp.password</name> + <value>anonymous@example.com</value> + <description>ftp login password.</description> +</property> + +<property> + <name>ftp.content.limit</name> + <value>65536</value> + <description>The length limit for downloaded content, in bytes. + If this value is nonnegative (>=0), content longer than it will be truncated; + otherwise, no truncation at all. + Caution: classical ftp RFCs never defines partial transfer and, in fact, + some ftp servers out there do not handle client side forced close-down very + well. Our implementation tries its best to handle such situations smoothly. + </description> +</property> + +<property> + <name>ftp.timeout</name> + <value>60000</value> + <description>Default timeout for ftp client socket, in millisec. + Please also see ftp.keep.connection below.</description> +</property> + +<property> + <name>ftp.server.timeout</name> + <value>100000</value> + <description>An estimation of ftp server idle time, in millisec. + Typically it is 120000 millisec for many ftp servers out there. + Better be conservative here. Together with ftp.timeout, it is used to + decide if we need to delete (annihilate) current ftp.client instance and + force to start another ftp.client instance anew. This is necessary because + a fetcher thread may not be able to obtain next request from queue in time + (due to idleness) before our ftp client times out or remote server + disconnects. Used only when ftp.keep.connection is true (please see below). + </description> +</property> + +<property> + <name>ftp.keep.connection</name> + <value>false</value> + <description>Whether to keep ftp connection. Useful if crawling same host + again and again. When set to true, it avoids connection, login and dir list + parser setup for subsequent urls. If it is set to true, however, you must + make sure (roughly): + (1) ftp.timeout is less than ftp.server.timeout + (2) ftp.timeout is larger than (fetcher.threads.fetch * fetcher.server.delay) + Otherwise there will be too many "delete client because idled too long" + messages in thread logs.</description> +</property> + +<property> + <name>ftp.follow.talk</name> + <value>false</value> + <description>Whether to log dialogue between our client and remote + server. Useful for debugging.</description> +</property> + +<!-- web db properties --> + +<property> + <name>db.default.fetch.interval</name> + <value>30</value> + <description>(DEPRECATED) The default number of days between re-fetches of a page. + </description> +</property> + +<property> + <name>db.fetch.interval.default</name> + <value>2592000</value> + <description>The default number of seconds between re-fetches of a page (30 days). + </description> +</property> + +<property> + <name>db.fetch.interval.max</name> + <value>7776000</value> + <description>The maximum number of seconds between re-fetches of a page + (90 days). After this period every page in the db will be re-tried, no + matter what is its status. + </description> +</property> + +<property> + <name>db.fetch.schedule.class</name> + <value>org.apache.nutch.crawl.DefaultFetchSchedule</value> + <description>The implementation of fetch schedule. DefaultFetchSchedule simply + adds the original fetchInterval to the last fetch time, regardless of + page changes.</description> +</property> + +<property> + <name>db.fetch.schedule.adaptive.inc_rate</name> + <value>0.4</value> + <description>If a page is unmodified, its fetchInterval will be + increased by this rate. This value should not + exceed 0.5, otherwise the algorithm becomes unstable.</description> +</property> + +<property> + <name>db.fetch.schedule.adaptive.dec_rate</name> + <value>0.2</value> + <description>If a page is modified, its fetchInterval will be + decreased by this rate. This value should not + exceed 0.5, otherwise the algorithm becomes unstable.</description> +</property> + +<property> + <name>db.fetch.schedule.adaptive.min_interval</name> + <value>60.0</value> + <description>Minimum fetchInterval, in seconds.</description> +</property> + +<property> + <name>db.fetch.schedule.adaptive.max_interval</name> + <value>31536000.0</value> + <description>Maximum fetchInterval, in seconds (365 days). + NOTE: this is limited by db.fetch.interval.max. Pages with + fetchInterval larger than db.fetch.interval.max + will be fetched anyway.</description> +</property> + +<property> + <name>db.fetch.schedule.adaptive.sync_delta</name> + <value>true</value> + <description>If true, try to synchronize with the time of page change. + by shifting the next fetchTime by a fraction (sync_rate) of the difference + between the last modification time, and the last fetch time.</description> +</property> + +<property> + <name>db.fetch.schedule.adaptive.sync_delta_rate</name> + <value>0.3</value> + <description>See sync_delta for description. This value should not + exceed 0.5, otherwise the algorithm becomes unstable.</description> +</property> + +<property> + <name>db.update.additions.allowed</name> + <value>true</value> + <description>If true, updatedb will add newly discovered URLs, if false + only already existing URLs in the CrawlDb will be updated and no new + URLs will be added. + </description> +</property> + +<property> + <name>db.ignore.internal.links</name> + <value>true</value> + <description>If true, when adding new links to a page, links from + the same host are ignored. This is an effective way to limit the + size of the link database, keeping only the highest quality + links. + </description> +</property> + +<property> + <name>db.ignore.external.links</name> + <value>false</value> + <description>If true, outlinks leading from a page to external hosts + will be ignored. This is an effective way to limit the crawl to include + only initially injected hosts, without creating complex URLFilters. + </description> +</property> + +<property> + <name>db.score.injected</name> + <value>1.0</value> + <description>The score of new pages added by the injector. + </description> +</property> + +<property> + <name>db.score.link.external</name> + <value>1.0</value> + <description>The score factor for new pages added due to a link from + another host relative to the referencing page's score. Scoring plugins + may use this value to affect initial scores of external links. + </description> +</property> + +<property> + <name>db.score.link.internal</name> + <value>1.0</value> + <description>The score factor for pages added due to a link from the + same host, relative to the referencing page's score. Scoring plugins + may use this value to affect initial scores of internal links. + </description> +</property> + +<property> + <name>db.score.count.filtered</name> + <value>false</value> + <description>The score value passed to newly discovered pages is + calculated as a fraction of the original page score divided by the + number of outlinks. If this option is false, only the outlinks that passed + URLFilters will count, if it's true then all outlinks will count. + </description> +</property> + +<property> + <name>db.max.inlinks</name> + <value>10000</value> + <description>Maximum number of Inlinks per URL to be kept in LinkDb. + If "invertlinks" finds more inlinks than this number, only the first + N inlinks will be stored, and the rest will be discarded. + </description> +</property> + +<property> + <name>db.max.outlinks.per.page</name> + <value>100</value> + <description>The maximum number of outlinks that we'll process for a page. + If this value is nonnegative (>=0), at most db.max.outlinks.per.page outlinks + will be processed for a page; otherwise, all outlinks will be processed. + </description> +</property> + +<property> + <name>db.max.anchor.length</name> + <value>100</value> + <description>The maximum number of characters permitted in an anchor. + </description> +</property> + +<property> + <name>db.fetch.retry.max</name> + <value>3</value> + <description>The maximum number of times a url that has encountered + recoverable errors is generated for fetch.</description> +</property> + +<property> + <name>db.signature.class</name> + <value>org.apache.nutch.crawl.MD5Signature</value> + <description>The default implementation of a page signature. Signatures + created with this implementation will be used for duplicate detection + and removal.</description> +</property> + +<property> + <name>db.signature.text_profile.min_token_len</name> + <value>2</value> + <description>Minimum token length to be included in the signature. + </description> +</property> + +<property> + <name>db.signature.text_profile.quant_rate</name> + <value>0.01</value> + <description>Profile frequencies will be rounded down to a multiple of + QUANT = (int)(QUANT_RATE * maxFreq), where maxFreq is a maximum token + frequency. If maxFreq > 1 then QUANT will be at least 2, which means that + for longer texts tokens with frequency 1 will always be discarded. + </description> +</property> + +<!-- generate properties --> + +<property> + <name>generate.max.per.host</name> + <value>-1</value> + <description>The maximum number of urls per host in a single + fetchlist. -1 if unlimited.</description> +</property> + +<property> + <name>generate.max.per.host.by.ip</name> + <value>false</value> + <description>If false, same host names are counted. If true, + hosts' IP addresses are resolved and the same IP-s are counted. + + -+-+-+- WARNING !!! -+-+-+- + When set to true, Generator will create a lot of DNS lookup + requests, rapidly. This may cause a DOS attack on + remote DNS servers, not to mention increased external traffic + and latency. For these reasons when using this option it is + required that a local caching DNS be used.</description> +</property> + +<property> + <name>generate.update.crawldb</name> + <value>false</value> + <description>For highly-concurrent environments, where several + generate/fetch/update cycles may overlap, setting this to true ensures + that generate will create different fetchlists even without intervening + updatedb-s, at the cost of running an additional job to update CrawlDB. + If false, running generate twice without intervening + updatedb will generate identical fetchlists.</description> +</property> + +<!-- fetcher properties --> + +<property> + <name>fetcher.server.delay</name> + <value>5.0</value> + <description>The number of seconds the fetcher will delay between + successive requests to the same server.</description> +</property> + +<property> + <name>fetcher.server.min.delay</name> + <value>0.0</value> + <description>The minimum number of seconds the fetcher will delay between + successive requests to the same server. This value is applicable ONLY + if fetcher.threads.per.host is greater than 1 (i.e. the host blocking + is turned off).</description> +</property> + +<property> + <name>fetcher.max.crawl.delay</name> + <value>30</value> + <description> + If the Crawl-Delay in robots.txt is set to greater than this value (in + seconds) then the fetcher will skip this page, generating an error report. + If set to -1 the fetcher will never skip such pages and will wait the + amount of time retrieved from robots.txt Crawl-Delay, however long that + might be. + </description> +</property> + +<property> + <name>fetcher.threads.fetch</name> + <value>10</value> + <description>The number of FetcherThreads the fetcher should use. + This is also determines the maximum number of requests that are + made at once (each FetcherThread handles one connection).</description> +</property> + +<property> + <name>fetcher.threads.per.host</name> + <value>1</value> + <description>This number is the maximum number of threads that + should be allowed to access a host at one time.</description> +</property> + +<property> + <name>fetcher.threads.per.host.by.ip</name> + <value>true</value> + <description>If true, then fetcher will count threads by IP address, + to which the URL's host name resolves. If false, only host name will be + used. NOTE: this should be set to the same value as + "generate.max.per.host.by.ip" - default settings are different only for + reasons of backward-compatibility.</description> +</property> + +<property> + <name>fetcher.verbose</name> + <value>false</value> + <description>If true, fetcher will log more verbosely.</description> +</property> + +<property> + <name>fetcher.parse</name> + <value>true</value> + <description>If true, fetcher will parse content.</description> +</property> + +<property> + <name>fetcher.store.content</name> + <value>true</value> + <description>If true, fetcher will store content.</description> +</property> + +<!-- indexer properties --> + +<property> + <name>indexer.score.power</name> + <value>0.5</value> + <description>Determines the power of link analyis scores. Each + pages's boost is set to <i>score<sup>scorePower</sup></i> where + <i>score</i> is its link analysis score and <i>scorePower</i> is the + value of this parameter. This is compiled into indexes, so, when + this is changed, pages must be re-indexed for it to take + effect.</description> +</property> + +<property> + <name>indexer.max.title.length</name> + <value>100</value> + <description>The maximum number of characters of a title that are indexed. + </description> +</property> + +<property> + <name>indexer.max.tokens</name> + <value>10000</value> + <description> + The maximum number of tokens that will be indexed for a single field + in a document. This limits the amount of memory required for + indexing, so that collections with very large files will not crash + the indexing process by running out of memory. + + Note that this effectively truncates large documents, excluding + from the index tokens that occur further in the document. If you + know your source documents are large, be sure to set this value + high enough to accomodate the expected size. If you set it to + -1, then the only limit is your memory, but you should anticipate + an OutOfMemoryError. + </description> +</property> + +<property> + <name>indexer.mergeFactor</name> + <value>50</value> + <description>The factor that determines the frequency of Lucene segment + merges. This must not be less than 2, higher values increase indexing + speed but lead to increased RAM usage, and increase the number of + open file handles (which may lead to "Too many open files" errors). + NOTE: the "segments" here have nothing to do with Nutch segments, they + are a low-level data unit used by Lucene. + </description> +</property> + +<property> + <name>indexer.minMergeDocs</name> + <value>50</value> + <description>This number determines the minimum number of Lucene + Documents buffered in memory between Lucene segment merges. Larger + values increase indexing speed and increase RAM usage. + </description> +</property> + +<property> + <name>indexer.maxMergeDocs</name> + <value>2147483647</value> + <description>This number determines the maximum number of Lucene + Documents to be merged into a new Lucene segment. Larger values + increase batch indexing speed and reduce the number of Lucene segments, + which reduces the number of open file handles; however, this also + decreases incremental indexing performance. + </description> +</property> + +<property> + <name>indexer.termIndexInterval</name> + <value>128</value> + <description>Determines the fraction of terms which Lucene keeps in + RAM when searching, to facilitate random-access. Smaller values use + more memory but make searches somewhat faster. Larger values use + less memory but make searches somewhat slower. + </description> +</property> + +<!-- indexingfilter plugin properties --> + +<property> + <name>indexingfilter.order</name> + <value></value> + <description>The order by which index filters are applied. + If empty, all available index filters (as dictated by properties + plugin-includes and plugin-excludes above) are loaded and applied in system + defined order. If not empty, only named filters are loaded and applied + in given order. For example, if this property has value: + org.apache.nutch.indexer.basic.BasicIndexingFilter org.apache.nutch.indexer.more.MoreIndexingFilter + then BasicIndexingFilter is applied first, and MoreIndexingFilter second. + + Filter ordering might have impact on result if one filter depends on output of + another filter. + </description> +</property> + + +<!-- analysis properties --> + +<property> + <name>analysis.common.terms.file</name> + <value>common-terms.utf8</value> + <description>The name of a file containing a list of common terms + that should be indexed in n-grams.</description> +</property> + +<!-- searcher properties --> + +<property> + <name>searcher.dir</name> + <value>crawl</value> + <description> + Path to root of crawl. This directory is searched (in + order) for either the file search-servers.txt, containing a list of + distributed search servers, or the directory "index" containing + merged indexes, or the directory "segments" containing segment + indexes. + </description> +</property> + +<property> + <name>searcher.filter.cache.size</name> + <value>16</value> + <description> + Maximum number of filters to cache. Filters can accelerate certain + field-based queries, like language, document format, etc. Each + filter requires one bit of RAM per page. So, with a 10 million page + index, a cache size of 16 consumes two bytes per page, or 20MB. + </description> +</property> + +<property> + <name>searcher.filter.cache.threshold</name> + <value>0.05</value> + <description> + Filters are cached when their term is matched by more than this + fraction of pages. For example, with a threshold of 0.05, and 10 + million pages, the term must match more than 1/20, or 50,000 pages. + So, if out of 10 million pages, 50% of pages are in English, and 2% + are in Finnish, then, with a threshold of 0.05, searches for + "lang:en" will use a cached filter, while searches for "lang:fi" + will score all 20,000 finnish documents. + </description> +</property> + +<property> + <name>searcher.hostgrouping.rawhits.factor</name> + <value>2.0</value> + <description> + A factor that is used to determine the number of raw hits + initially fetched, before host grouping is done. + </description> +</property> + +<property> + <name>searcher.summary.context</name> + <value>5</value> + <description> + The number of context terms to display preceding and following + matching terms in a hit summary. + </description> +</property> + +<property> + <name>searcher.summary.length</name> + <value>20</value> + <description> + The total number of terms to display in a hit summary. + </description> +</property> + +<property> + <name>searcher.max.hits</name> + <value>-1</value> + <description>If positive, search stops after this many hits are + found. Setting this to small, positive values (e.g., 1000) can make + searches much faster. With a sorted index, the quality of the hits + suffers little.</description> +</property> + +<property> + <name>searcher.max.time.tick_count</name> + <value>-1</value> + <description>If positive value is defined here, limit search time for + every request to this number of elapsed ticks (see the tick_length + property below). The total maximum time for any search request will be + then limited to tick_count * tick_length milliseconds. When search time + is exceeded, partial results will be returned, and the total number of + hits will be estimated. + </description> +</property> + +<property> + <name>searcher.max.time.tick_length</name> + <value>200</value> + <description>The number of milliseconds between ticks. Larger values + reduce the timer granularity (precision). Smaller values bring more + overhead. + </description> +</property> + +<property> + <name>searcher.num.handlers</name> + <value>10</value> + <description>The number of handlers for the distributed search server. + </description> +</property> + +<property> + <name>searcher.max.hits.per.page</name> + <value>1000</value> + <description> The maximum number of hits to show per page. -1 if + unlimited. If the number of hits requested by the user (via + hitsPerPage parameter in the query string) is more than the value + specified in this property, then this value is assumed as the number + of hits per page. + </description> +</property> + +<!-- URL normalizer properties --> + +<property> + <name>urlnormalizer.order</name> + <value>org.apache.nutch.net.urlnormalizer.basic.BasicURLNormalizer org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer</value> + <description>Order in which normalizers will run. If any of these isn't + activated it will be silently skipped. If other normalizers not on the + list are activated, they will run in random order after the ones + specified here are run. + </description> +</property> + +<property> + <name>urlnormalizer.regex.file</name> + <value>regex-normalize.xml</value> + <description>Name of the config file used by the RegexUrlNormalizer class. + </description> +</property> + +<property> + <name>urlnormalizer.loop.count</name> + <value>1</value> + <description>Optionally loop through normalizers several times, to make + sure that all transformations have been performed. + </description> +</property> + +<!-- mime properties --> + +<property> + <name>mime.types.file</name> + <value>tika-mimetypes.xml</value> + <description>Name of file in CLASSPATH containing filename extension and + magic sequence to mime types mapping information</description> +</property> + +<property> + <name>mime.type.magic</name> + <value>true</value> + <description>Defines if the mime content type detector uses magic resolution. + </description> +</property> + +<!-- plugin properties --> + +<property> + <name>plugin.folders</name> + <value>plugins</value> + <description>Directories where nutch plugins are located. Each + element may be a relative or absolute path. If absolute, it is used + as is. If relative, it is searched for on the classpath.</description> +</property> + +<property> + <name>plugin.auto-activation</name> + <value>true</value> + <description>Defines if some plugins that are not activated regarding + the plugin.includes and plugin.excludes properties must be automaticaly + activated if they are needed by some actived plugins. + </description> +</property> + +<property> + <name>plugin.includes</name> + <value>protocol-http|urlfilter-regex|parse-(text|html|js)|index-(basic|anchor)|query-(basic|site|url)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic)</value> + <description>Regular expression naming plugin directory names to + include. Any plugin not matching this expression is excluded. + In any case you need at least include the nutch-extensionpoints plugin. By + default Nutch includes crawling just HTML and plain text via HTTP, + and basic indexing and search plugins. In order to use HTTPS please enable + protocol-httpclient, but be aware of possible intermittent problems with the + underlying commons-httpclient library. + </description> +</property> + +<property> + <name>plugin.excludes</name> + <value></value> + <description>Regular expression naming plugin directory names to exclude. + </description> +</property> + +<!-- parser properties --> + +<property> + <name>parse.plugin.file</name> + <value>parse-plugins.xml</value> + <description>The name of the file that defines the associations between + content-types and parsers.</description> +</property> + +<property> + <name>parser.character.encoding.default</name> + <value>windows-1252</value> + <description>The character encoding to fall back to when no other information + is available</description> +</property> + +<property> + <name>encodingdetector.charset.min.confidence</name> + <value>-1</value> + <description>A integer between 0-100 indicating minimum confidence value + for charset auto-detection. Any negative value disables auto-detection. + </description> +</property> + +<property> + <name>parser.caching.forbidden.policy</name> + <value>content</value> + <description>If a site (or a page) requests through its robot metatags + that it should not be shown as cached content, apply this policy. Currently + three keywords are recognized: "none" ignores any "noarchive" directives. + "content" doesn't show the content, but shows summaries (snippets). + "all" doesn't show either content or summaries.</description> +</property> + + +<property> + <name>parser.html.impl</name> + <value>neko</value> + <description>HTML Parser implementation. Currently the following keywords + are recognized: "neko" uses NekoHTML, "tagsoup" uses TagSoup. + </description> +</property> + +<property> + <name>parser.html.form.use_action</name> + <value>false</value> + <description>If true, HTML parser will collect URLs from form action + attributes. This may lead to undesirable behavior (submitting empty + forms during next fetch cycle). If false, form action attribute will + be ignored.</description> +</property> + +<property> + <name>parser.html.outlinks.ignore_tags</name> + <value></value> + <description>Comma separated list of HTML tags, from which outlinks + shouldn't be extracted. Nutch takes links from: a, area, form, frame, + iframe, script, link, img. If you add any of those tags here, it + won't be taken. Default is empty list. Probably reasonable value + for most people would be "img,script,link".</description> +</property> + + +<!-- urlfilter plugin properties --> + +<property> + <name>urlfilter.regex.file</name> + <value>regex-urlfilter.txt</value> + <description>Name of file on CLASSPATH containing regular expressions + used by urlfilter-regex (RegexURLFilter) plugin.</description> +</property> + +<property> + <name>urlfilter.automaton.file</name> + <value>automaton-urlfilter.txt</value> + <description>Name of file on CLASSPATH containing regular expressions + used by urlfilter-automaton (AutomatonURLFilter) plugin.</description> +</property> + +<property> + <name>urlfilter.prefix.file</name> + <value>prefix-urlfilter.txt</value> + <description>Name of file on CLASSPATH containing url prefixes + used by urlfilter-prefix (PrefixURLFilter) plugin.</description> +</property> + +<property> + <name>urlfilter.suffix.file</name> + <value>suffix-urlfilter.txt</value> + <description>Name of file on CLASSPATH containing url suffixes + used by urlfilter-suffix (SuffixURLFilter) plugin.</description> +</property> + +<property> + <name>urlfilter.order</name> + <value></value> + <description>The order by which url filters are applied. + If empty, all available url filters (as dictated by properties + plugin-includes and plugin-excludes above) are loaded and applied in system + defined order. If not empty, only named filters are loaded and applied + in given order. For example, if this property has value: + org.apache.nutch.urlfilter.regex.RegexURLFilter org.apache.nutch.urlfilter.prefix.PrefixURLFilter + then RegexURLFilter is applied first, and PrefixURLFilter second. + Since all filters are AND'ed, filter ordering does not have impact + on end result, but it may have performance implication, depending + on relative expensiveness of filters. + </description> +</property> + +<!-- scoring filters properties --> + +<property> + <name>scoring.filter.order</name> + <value></value> + <description>The order in which scoring filters are applied. + This may be left empty (in which case all available scoring + filters will be applied in the order defined in plugin-includes + and plugin-excludes), or a space separated list of implementation + classes. + </description> +</property> + +<!-- clustering extension properties --> + +<property> + <name>extension.clustering.hits-to-cluster</name> + <value>100</value> + <description>Number of snippets retrieved for the clustering extension + if clustering extension is available and user requested results + to be clustered.</description> +</property> + +<property> + <name>extension.clustering.extension-name</name> + <value></value> + <description>Use the specified online clustering extension. If empty, + the first available extension will be used. The "name" here refers to an 'id' + attribute of the 'implementation' element in the plugin descriptor XML + file.</description> +</property> + +<!-- ontology extension properties --> + +<property> + <name>extension.ontology.extension-name</name> + <value></value> + <description>Use the specified online ontology extension. If empty, + the first available extension will be used. The "name" here refers to an 'id' + attribute of the 'implementation' element in the plugin descriptor XML + file.</description> +</property> + +<property> + <name>extension.ontology.urls</name> + <value> + </value> + <description>Urls of owl files, separated by spaces, such as + http://www.example.com/ontology/time.owl + http://www.example.com/ontology/space.owl + http://www.example.com/ontology/wine.owl + Or + file:/ontology/time.owl + file:/ontology/space.owl + file:/ontology/wine.owl + You have to make sure each url is valid. + By default, there is no owl file, so query refinement based on ontology + is silently ignored. + </description> +</property> + +<!-- query-basic plugin properties --> + +<property> + <name>query.url.boost</name> + <value>4.0</value> + <description> Used as a boost for url field in Lucene query. + </description> +</property> + +<property> + <name>query.anchor.boost</name> + <value>2.0</value> + <description> Used as a boost for anchor field in Lucene query. + </description> +</property> + +<property> + <name>query.title.boost</name> + <value>1.5</value> + <description> Used as a boost for title field in Lucene query. + </description> +</property> + +<property> + <name>query.host.boost</name> + <value>2.0</value> + <description> Used as a boost for host field in Lucene query. + </description> +</property> + +<property> + <name>query.phrase.boost</name> + <value>1.0</value> + <description> Used as a boost for phrase in Lucene query. + Multiplied by boost for field phrase is matched in. + </description> +</property> + +<!-- creative-commons plugin properties --> + +<property> + <name>query.cc.boost</name> + <value>0.0</value> + <description> Used as a boost for cc field in Lucene query. + </description> +</property> + +<!-- query-more plugin properties --> + +<property> + <name>query.type.boost</name> + <value>0.0</value> + <description> Used as a boost for type field in Lucene query. + </description> +</property> + +<!-- query-site plugin properties --> + +<property> + <name>query.site.boost</name> + <value>0.0</value> + <description> Used as a boost for site field in Lucene query. + </description> +</property> + +<!-- microformats-reltag plugin properties --> + +<property> + <name>query.tag.boost</name> + <value>1.0</value> + <description> Used as a boost for tag field in Lucene query. + </description> +</property> + +<!-- language-identifier plugin properties --> + +<property> + <name>lang.ngram.min.length</name> + <value>1</value> + <description> The minimum size of ngrams to uses to identify + language (must be between 1 and lang.ngram.max.length). + The larger is the range between lang.ngram.min.length and + lang.ngram.max.length, the better is the identification, but + the slowest it is. + </description> +</property> + +<property> + <name>lang.ngram.max.length</name> + <value>4</value> + <description> The maximum size of ngrams to uses to identify + language (must be between lang.ngram.min.length and 4). + The larger is the range between lang.ngram.min.length and + lang.ngram.max.length, the better is the identification, but + the slowest it is. + </description> +</property> + +<property> + <name>lang.analyze.max.length</name> + <value>2048</value> + <description> The maximum bytes of data to uses to indentify + the language (0 means full content analysis). + The larger is this value, the better is the analysis, but the + slowest it is. + </description> +</property> + +<property> + <name>query.lang.boost</name> + <value>0.0</value> + <description> Used as a boost for lang field in Lucene query. + </description> +</property> + +<!-- Temporary Hadoop 0.17.x workaround. --> + +<property> + <name>hadoop.job.history.user.location</name> + <value>${hadoop.log.dir}/history/user</value> + <description>Hadoop 0.17.x comes with a default setting to create + user logs inside the output path of the job. This breaks some + Hadoop classes, which expect the output to contain only + part-XXXXX files. This setting changes the output to a + subdirectory of the regular log directory. + </description> +</property> + +</configuration> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-site.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-site.xml new file mode 100644 index 0000000..70975c2 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/nutch-site.xml @@ -0,0 +1,58 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + +<property> + <name>http.agent.name</name> + <value>Spacewalk</value> + <description>HTTP 'User-Agent' request header. MUST NOT be empty - + please set this to a single word uniquely related to your organization. + + NOTE: You should also check other related properties: + + http.robots.agents + http.agent.description + http.agent.url + http.agent.email + http.agent.version + + and set their values appropriately. + + </description> +</property> + +<property> + <name>fetcher.server.delay</name> + <value>0</value> + <description>The number of seconds the fetcher will delay between + successive requests to the same server.</description> +</property> + +<property> + <name>fetcher.threads.per.host</name> + <value>50</value> + <description>This number is the maximum number of threads that + should be allowed to access a host at one time.</description> +</property> + +<property> + <name>plugin.folders</name> + <value>/usr/share/nutch/plugins</value> + <description>Directories where nutch plugins are located. Each + element may be a relative or absolute path. If absolute, it is used + as is. If relative, it is searched for on the classpath.</description> +</property> + +<property> +<name>plugin.includes</name> + <value>protocol-file|urlfilter-regex|parse-(xml|text|html|js|pdf)|index-basic|query-(basic|site|url)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic)</value> +</property> + +<property> + <name>file.content.limit</name> <value>-1</value> +</property> + +</configuration> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.dtd b/search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.dtd new file mode 100644 index 0000000..ae21045 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.dtd @@ -0,0 +1,12 @@ +<!ELEMENT parse-plugins (mimeType+,aliases)> +<!ELEMENT mimeType (plugin+)> +<!ATTLIST mimeType name CDATA #REQUIRED> + +<!ELEMENT plugin EMPTY> +<!ATTLIST plugin id CDATA #REQUIRED> +<!ATTLIST plugin order CDATA ''> + +<!ELEMENT aliases (alias+)> +<!ELEMENT alias EMPTY> +<!ATTLIST alias name CDATA #REQUIRED> +<!ATTLIST alias extension-id CDATA #REQUIRED> \ No newline at end of file diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.xml new file mode 100644 index 0000000..876d188 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/parse-plugins.xml @@ -0,0 +1,221 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author : mattmann + Description: This xml file represents a natural ordering for which parsing + plugin should get called for a particular mimeType. +--> + +<parse-plugins> + + <mimeType name="application/msword"> + <plugin id="parse-msword" /> + </mimeType> + + <mimeType name="application/pdf"> + <plugin id="parse-pdf" /> + </mimeType> + + <mimeType name="application/postscript"> + <plugin id="parse-pdf" /> + </mimeType> + + <mimeType name="application/rss+xml"> + <plugin id="parse-rss" /> + <plugin id="feed" /> + </mimeType> + + <mimeType name="application/vnd.ms-excel"> + <plugin id="parse-msexcel" /> + </mimeType> + + <mimeType name="application/vnd.ms-powerpoint"> + <plugin id="parse-mspowerpoint" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.text"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.text-template"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.text-master"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.text-web"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.presentation"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.presentation-template"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.spreadsheet"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.oasis.opendocument.spreadsheet-template"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.sun.xml.calc"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.sun.xml.calc.template"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.sun.xml.impress"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.sun.xml.impress.template"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.sun.xml.writer"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/vnd.sun.xml.writer.template"> + <plugin id="parse-oo" /> + </mimeType> + + <mimeType name="application/xhtml+xml"> + <plugin id="parse-html" /> + </mimeType> + + <mimeType name="application/x-bzip2"> + <!-- try and parse it with the zip parser --> + <plugin id="parse-zip" /> + </mimeType> + + <mimeType name="application/x-csh"> + <plugin id="parse-text" /> + </mimeType> + + <mimeType name="application/x-gzip"> + <!-- try and parse it with the zip parser --> + <plugin id="parse-zip" /> + </mimeType> + + <mimeType name="application/x-javascript"> + <plugin id="parse-js" /> + </mimeType> + + <mimeType name="application/x-kword"> + <!-- try and parse it with the word parser --> + <plugin id="parse-msword" /> + </mimeType> + + <mimeType name="application/x-kspread"> + <!-- try and parse it with the msexcel parser --> + <plugin id="parse-msexcel" /> + </mimeType> + + <mimeType name="application/x-shockwave-flash"> + <plugin id="parse-swf" /> + </mimeType> + + <mimeType name="application/zip"> + <plugin id="parse-zip" /> + </mimeType> + + <mimeType name="text/html"> + <plugin id="parse-html" /> + </mimeType> + + <mimeType name="text/plain"> + <plugin id="parse-text" /> + </mimeType> + + <mimeType name="text/richtext"> + <plugin id="parse-rtf" /> + <plugin id="parse-msword" /> + </mimeType> + + <mimeType name="text/rtf"> + <plugin id="parse-rtf" /> + <plugin id="parse-msword" /> + </mimeType> + + <mimeType name="text/sgml"> + <plugin id="parse-html" /> + </mimeType> + + <mimeType name="text/tab-separated-values"> + <plugin id="parse-msexcel" /> + </mimeType> + + <mimeType name="text/xml"> + <plugin id="parse-html" /> + <plugin id="parse-rss" /> + <plugin id="feed" /> + </mimeType> + + <!-- Types for parse-ext plugin: required for unit tests to pass. --> + + <mimeType name="application/vnd.nutch.example.cat"> + <plugin id="parse-ext" /> + </mimeType> + + <mimeType name="application/vnd.nutch.example.md5sum"> + <plugin id="parse-ext" /> + </mimeType> + + <!-- alias mappings for parse-xxx names to the actual extension implementation + ids described in each plugin's plugin.xml file --> + <aliases> + <alias name="parse-ext" extension-id="ExtParser" /> + <alias name="parse-html" + extension-id="org.apache.nutch.parse.html.HtmlParser" /> + <alias name="parse-js" extension-id="JSParser" /> + <alias name="parse-mp3" + extension-id="org.apache.nutch.parse.mp3.MP3Parser" /> + <alias name="parse-msexcel" + extension-id="org.apache.nutch.parse.msexcel.MSExcelParser" /> + <alias name="parse-mspowerpoint" + extension-id="org.apache.nutch.parse.mspowerpoint.MSPowerPointParser" /> + <alias name="parse-msword" + extension-id="org.apache.nutch.parse.msword.MSWordParser" /> + <alias name="parse-oo" + extension-id="org.apache.nutch.parse.oo.OpenDocument.Text" /> + <alias name="parse-pdf" + extension-id="org.apache.nutch.parse.pdf.PdfParser" /> + <alias name="parse-rss" + extension-id="org.apache.nutch.parse.rss.RSSParser" /> + <alias name="feed" + extension-id="org.apache.nutch.parse.feed.FeedParser" /> + <alias name="parse-rtf" + extension-id="org.apache.nutch.parse.rtf.RTFParseFactory" /> + <alias name="parse-swf" + extension-id="org.apache.nutch.parse.swf.SWFParser" /> + <alias name="parse-text" + extension-id="org.apache.nutch.parse.text.TextParser" /> + <alias name="parse-zip" + extension-id="org.apache.nutch.parse.zip.ZipParser" /> + </aliases> + +</parse-plugins> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/prefix-urlfilter.txt b/search-server/spacewalk-search/nutch/crawl_jsp/conf/prefix-urlfilter.txt new file mode 100644 index 0000000..491dc1f --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/prefix-urlfilter.txt @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# config file for urlfilter-prefix plugin + +http:// +https:// +ftp:// +file:// diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-normalize.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-normalize.xml new file mode 100644 index 0000000..8d8bad1 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-normalize.xml @@ -0,0 +1,66 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- This is the configuration file for the RegexUrlNormalize Class. + This is intended so that users can specify substitutions to be + done on URLs. The regex engine that is used is Perl5 compatible. + The rules are applied to URLs in the order they occur in this file. --> + +<!-- WATCH OUT: an xml parser reads this file an ampersands must be + expanded to & --> + +<!-- The following rules show how to strip out session IDs, default pages, + interpage anchors, etc. Order does matter! --> +<regex-normalize> + +<!-- removes session ids from urls (such as jsessionid and PHPSESSID) --> +<regex> + <pattern>([;_]?((?i)l|j|bv_)?((?i)sid|phpsessid|sessionid)=.*?)(?|&|#|$)</pattern> + <substitution>$4</substitution> +</regex> + +<!-- changes default pages into standard for /index.html, etc. into / +<regex> + <pattern>/((?i)index|default)\.((?i)js[pf]{1}?[afx]?|cgi|cfm|asp[x]?|[psx]?htm[l]?|php[3456]?)(\?|&|#|$)</pattern> + <substitution>/$3</substitution> +</regex> --> + +<!-- removes interpage href anchors such as site.com#location --> +<regex> + <pattern>#.*?(?|&|$)</pattern> + <substitution>$1</substitution> +</regex> + +<!-- cleans ?&var=value into ?var=value --> +<regex> + <pattern>?&</pattern> + <substitution>?</substitution> +</regex> + +<!-- cleans multiple sequential ampersands into a single ampersand --> +<regex> + <pattern>&{2,}</pattern> + <substitution>&</substitution> +</regex> + +<!-- removes trailing ? --> +<regex> + <pattern>[?&.]$</pattern> + <substitution></substitution> +</regex> + +</regex-normalize> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-urlfilter.txt b/search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-urlfilter.txt new file mode 100644 index 0000000..fe56603 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/regex-urlfilter.txt @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The default url filter. +# Better for whole-internet crawling. + +# Each non-comment, non-blank line contains a regular expression +# prefixed by '+' or '-'. The first matching pattern in the file +# determines whether a URL is included or ignored. If no pattern +# matches, the URL is ignored. + +# skip file: ftp: and mailto: urls +-^(file|ftp|mailto): + +# skip image and other suffixes we can't yet parse +-.(gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|jpeg|JPEG|bmp|BMP)$ + +# skip URLs containing certain characters as probable queries, etc. +-[?*!@=] + +# skip URLs with slash-delimited segment that repeats 3+ times, to break loops +-.*(/[^/]+)/[^/]+\1/[^/]+\1/ + +# accept anything else ++. diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/slaves b/search-server/spacewalk-search/nutch/crawl_jsp/conf/slaves new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/slaves @@ -0,0 +1 @@ +localhost diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/subcollections.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/subcollections.xml new file mode 100644 index 0000000..7b8805d --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/subcollections.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<subcollections> + <subcollection> + <name>nutch</name> + <id>nutch</id> + <whitelist> +http://lucene.apache.org/nutch/ +http://wiki.apache.org/nutch/ + </whitelist> + <blacklist /> + </subcollection> +</subcollections> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/suffix-urlfilter.txt b/search-server/spacewalk-search/nutch/crawl_jsp/conf/suffix-urlfilter.txt new file mode 100644 index 0000000..b510d4a --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/suffix-urlfilter.txt @@ -0,0 +1,91 @@ +# config file for urlfilter-suffix plugin + +# case-insensitive, allow unknown suffixes ++I +# uncomment the line below to filter on url path +#+P + +### prohibit these +# pictures +.gif +.jpg +.jpeg +.bmp +.png +.tif +.tiff +.ico +.eps +.ps +.wmf +.fpx +.cur +.ani +.img +.lwf +.pcd +.psp +.psd +.tga +.xbm +.xpm + +# web-formats +.css + +# archives/packages +.arj +.arc +.7z +.cab +.lzw +.lha +.lzh +.zip +.gz +.tar +.tgz +.sit +.rpm +.deb +.pkg + +# audio/video +.mid +.midi +.rmi +.mpeg +.mpg +.mpe +.mp3 +.mp2 +.aac +.mov +.fla +.flv +.ra +.ram +.rm +.rmv +.wma +.wmv +.wav +.wave +.ogg +.avi +.au +.snd + +# executables +.exe +.com + +# windows links +.lnk + +# typo3-extensions +.t3x + +# disc-images +.iso +.bin diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/conf/tika-mimetypes.xml b/search-server/spacewalk-search/nutch/crawl_jsp/conf/tika-mimetypes.xml new file mode 100644 index 0000000..ce84078 --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/conf/tika-mimetypes.xml @@ -0,0 +1,367 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Description: This xml file defines the valid mime types used by Tika. + The mime types within this file are based on the types in the mime-types.xml + file available in Apache Nutch. +--> + +<mime-info> + + <mime-type type="text/plain"> + <magic priority="50"> + <match value="This is TeX," type="string" offset="0" /> + <match value="This is METAFONT," type="string" offset="0" /> + </magic> + <glob pattern="*.txt" /> + <glob pattern="*.asc" /> + </mime-type> + + <mime-type type="text/html"> + <magic priority="50"> + <match value="<!DOCTYPE HTML" type="string" + offset="0:64" /> + <match value="<!doctype html" type="string" + offset="0:64" /> + <match value="<HEAD" type="string" offset="0:64" /> + <match value="<head" type="string" offset="0:64" /> + <match value="<TITLE" type="string" offset="0:64" /> + <match value="<title" type="string" offset="0:64" /> + <match value="<html" type="string" offset="0:64" /> + <match value="<HTML" type="string" offset="0:64" /> + <match value="<BODY" type="string" offset="0" /> + <match value="<body" type="string" offset="0" /> + <match value="<TITLE" type="string" offset="0" /> + <match value="<title" type="string" offset="0" /> + <match value="<!--" type="string" offset="0" /> + <match value="<h1" type="string" offset="0" /> + <match value="<H1" type="string" offset="0" /> + <match value="<!doctype HTML" type="string" offset="0" /> + <match value="<!DOCTYPE html" type="string" offset="0" /> + </magic> + <glob pattern="*.html" /> + <glob pattern="*.htm" /> + </mime-type> + + <mime-type type="application/xhtml+xml"> + <glob pattern="*.xhtml" /> + <root-XML namespaceURI='http://www.w3.org/1999/xhtml' + localName='html' /> + </mime-type> + + <mime-type type="application/vnd.ms-powerpoint"> + <glob pattern="*.ppz" /> + <glob pattern="*.ppt" /> + <glob pattern="*.pps" /> + <glob pattern="*.pot" /> + <magic priority="50"> + <match value="0xcfd0e011" type="little32" offset="0" /> + </magic> + </mime-type> + + <mime-type type="application/vnd.ms-excel"> + <magic priority="50"> + <match value="Microsoft Excel 5.0 Worksheet" type="string" + offset="2080" /> + </magic> + <glob pattern="*.xls" /> + <glob pattern="*.xlc" /> + <glob pattern="*.xll" /> + <glob pattern="*.xlm" /> + <glob pattern="*.xlw" /> + <glob pattern="*.xla" /> + <glob pattern="*.xlt" /> + <glob pattern="*.xld" /> + <alias type="application/msexcel" /> + </mime-type> + + <mime-type type="application/vnd.oasis.opendocument.text"> + <glob pattern="*.odt" /> + </mime-type> + + + <mime-type type="application/zip"> + <alias type="application/x-zip-compressed" /> + <magic priority="40"> + <match value="PK\003\004" type="string" offset="0" /> + </magic> + <glob pattern="*.zip" /> + </mime-type> + + <mime-type type="application/vnd.oasis.opendocument.text"> + <glob pattern="*.oth" /> + </mime-type> + + <mime-type type="application/msword"> + <magic priority="50"> + <match value="\x31\xbe\x00\x00" type="string" offset="0" /> + <match value="PO^Q`" type="string" offset="0" /> + <match value="\376\067\0\043" type="string" offset="0" /> + <match value="\333\245-\0\0\0" type="string" offset="0" /> + <match value="Microsoft Word 6.0 Document" type="string" + offset="2080" /> + <match value="Microsoft Word document data" type="string" + offset="2112" /> + </magic> + <glob pattern="*.doc" /> + <alias type="application/vnd.ms-word" /> + </mime-type> + + <mime-type type="application/octet-stream"> + <magic priority="50"> + <match value="\037\036" type="string" offset="0" /> + <match value="017437" type="host16" offset="0" /> + <match value="0x1fff" type="host16" offset="0" /> + <match value="\377\037" type="string" offset="0" /> + <match value="0145405" type="host16" offset="0" /> + </magic> + <glob pattern="*.bin" /> + </mime-type> + + <mime-type type="application/pdf"> + <magic priority="50"> + <match value="%PDF-" type="string" offset="0" /> + </magic> + <glob pattern="*.pdf" /> + <alias type="application/x-pdf" /> + </mime-type> + + <mime-type type="application/atom+xml"> + <root-XML localName="feed" + namespaceURI="http://purl.org/atom/ns#" /> + </mime-type> + + <mime-type type="application/mac-binhex40"> + <glob pattern="*.hqx" /> + </mime-type> + + <mime-type type="application/mac-compactpro"> + <glob pattern="*.cpt" /> + </mime-type> + + <mime-type type="application/rtf"> + <glob pattern="*.rtf"/> + <alias type="text/rtf" /> + </mime-type> + + <mime-type type="application/rss+xml"> + <alias type="text/rss" /> + <root-XML localName="rss" /> + <root-XML namespaceURI="http://purl.org/rss/1.0/" /> + <glob pattern="*.rss" /> + </mime-type> + + <!-- added in by mattmann --> + <mime-type type="application/xml"> + <alias type="text/xml" /> + <glob pattern="*.xml" /> + </mime-type> + + <mime-type type="application/x-mif"> + <alias type="application/vnd.mif" /> + </mime-type> + + <mime-type type="application/vnd.wap.wbxml"> + <glob pattern="*.wbxml" /> + </mime-type> + + <mime-type type="application/vnd.wap.wmlc"> + <_comment>Compiled WML Document</_comment> + <glob pattern="*.wmlc" /> + </mime-type> + + <mime-type type="application/vnd.wap.wmlscriptc"> + <_comment>Compiled WML Script</_comment> + <glob pattern="*.wmlsc" /> + </mime-type> + + <mime-type type="text/vnd.wap.wmlscript"> + <_comment>WML Script</_comment> + <glob pattern="*.wmls" /> + </mime-type> + + <mime-type type="application/x-bzip"> + <alias type="application/x-bzip2" /> + </mime-type> + + <mime-type type="application/x-bzip-compressed-tar"> + <glob pattern="*.tbz" /> + <glob pattern="*.tbz2" /> + </mime-type> + + <mime-type type="application/x-cdlink"> + <_comment>Virtual CD-ROM CD Image File</_comment> + <glob pattern="*.vcd" /> + </mime-type> + + <mime-type type="application/x-director"> + <_comment>Shockwave Movie</_comment> + <glob pattern="*.dcr" /> + <glob pattern="*.dir" /> + <glob pattern="*.dxr" /> + </mime-type> + + <mime-type type="application/x-futuresplash"> + <_comment>Macromedia FutureSplash File</_comment> + <glob pattern="*.spl" /> + </mime-type> + + <mime-type type="application/x-java"> + <alias type="application/java" /> + </mime-type> + + <mime-type type="application/x-koan"> + <_comment>SSEYO Koan File</_comment> + <glob pattern="*.skp" /> + <glob pattern="*.skd" /> + <glob pattern="*.skt" /> + <glob pattern="*.skm" /> + </mime-type> + + <mime-type type="application/x-latex"> + <_comment>LaTeX Source Document</_comment> + <glob pattern="*.latex" /> + </mime-type> + + <!-- JC CHANGED + <mime-type type="application/x-mif"> + <_comment>FrameMaker MIF document</_comment> + <glob pattern="*.mif"/> + </mime-type> --> + + <mime-type type="application/x-ms-dos-executable"> + <alias type="application/x-dosexec" /> + </mime-type> + + <mime-type type="application/ogg"> + <alias type="application/x-ogg" /> + </mime-type> + + <mime-type type="application/x-rar"> + <alias type="application/x-rar-compressed" /> + </mime-type> + + <mime-type type="application/x-shellscript"> + <alias type="application/x-sh" /> + </mime-type> + + <mime-type type="application/xhtml+xml"> + <glob pattern="*.xht" /> + </mime-type> + + <mime-type type="audio/midi"> + <glob pattern="*.kar" /> + </mime-type> + + <mime-type type="audio/x-pn-realaudio"> + <alias type="audio/x-realaudio" /> + </mime-type> + + <mime-type type="image/tiff"> + <magic priority="50"> + <match value="0x4d4d2a00" type="string" offset="0" /> + <match value="0x49492a00" type="string" offset="0" /> + </magic> + </mime-type> + + <mime-type type="message/rfc822"> + <magic priority="50"> + <match type="string" value="Relay-Version:" offset="0" /> + <match type="string" value="#! rnews" offset="0" /> + <match type="string" value="N#! rnews" offset="0" /> + <match type="string" value="Forward to" offset="0" /> + <match type="string" value="Pipe to" offset="0" /> + <match type="string" value="Return-Path:" offset="0" /> + <match type="string" value="From:" offset="0" /> + <match type="string" value="Message-ID:" offset="0" /> + <match type="string" value="Date:" offset="0" /> + </magic> + </mime-type> + + <mime-type type="application/x-javascript"> + <glob pattern="*.js" /> + </mime-type> + + + <mime-type type="image/vnd.wap.wbmp"> + <_comment>Wireless Bitmap File Format</_comment> + <glob pattern="*.wbmp" /> + </mime-type> + + <mime-type type="image/x-psd"> + <alias type="image/photoshop" /> + </mime-type> + + <mime-type type="image/x-xcf"> + <alias type="image/xcf" /> + <magic priority="50"> + <match type="string" value="gimp xcf " offset="0" /> + </magic> + </mime-type> + + <mime-type type="application/x-shockwave-flash"> + <glob pattern="*.swf"/> + <magic priority="50"> + <match type="string" value="FWS" offset="0"/> + <match type="string" value="CWS" offset="0"/> + </magic> + </mime-type> + + <mime-type type="model/iges"> + <_comment> + Initial Graphics Exchange Specification Format + </_comment> + <glob pattern="*.igs" /> + <glob pattern="*.iges" /> + </mime-type> + + <mime-type type="model/mesh"> + <glob pattern="*.msh" /> + <glob pattern="*.mesh" /> + <glob pattern="*.silo" /> + </mime-type> + + <mime-type type="model/vrml"> + <glob pattern="*.vrml" /> + </mime-type> + + <mime-type type="text/x-tcl"> + <alias type="application/x-tcl" /> + </mime-type> + + <mime-type type="text/x-tex"> + <alias type="application/x-tex" /> + </mime-type> + + <mime-type type="text/x-texinfo"> + <alias type="application/x-texinfo" /> + </mime-type> + + <mime-type type="text/x-troff-me"> + <alias type="application/x-troff-me" /> + </mime-type> + + <mime-type type="video/vnd.mpegurl"> + <glob pattern="*.mxu" /> + </mime-type> + + <mime-type type="x-conference/x-cooltalk"> + <_comment>Cooltalk Audio</_comment> + <glob pattern="*.ice" /> + </mime-type> + +</mime-info> diff --git a/search-server/spacewalk-search/nutch/crawl_jsp/urls/urls.txt b/search-server/spacewalk-search/nutch/crawl_jsp/urls/urls.txt new file mode 100644 index 0000000..020ed3b --- /dev/null +++ b/search-server/spacewalk-search/nutch/crawl_jsp/urls/urls.txt @@ -0,0 +1,7 @@ +file:///shared/repo/trunk/eng/docs/guides/channel-mgmt/en/ +file:///shared/repo/trunk/eng/docs/guides/client-config/en/ +file:///shared/repo/trunk/eng/docs/guides/proxy/rhn511/en/ +file:///shared/repo/trunk/eng/docs/guides/reference/en/ +file:///shared/repo/trunk/eng/docs/guides/satellite/en/ +file:///shared/repo/trunk/eng/docs/release-notes/satellite/rhn510/en-US/ + diff --git a/search-server/spacewalk-search/rhn_search_daemon_dev.conf b/search-server/spacewalk-search/rhn_search_daemon_dev.conf new file mode 100644 index 0000000..964fe46 --- /dev/null +++ b/search-server/spacewalk-search/rhn_search_daemon_dev.conf @@ -0,0 +1,45 @@ +wrapper.java.command=/usr/bin/java +wrapper.java.mainclass=com.redhat.satellite.search.Main +wrapper.java.classpath.1=%PWD%/build/ +wrapper.java.classpath.2=%PWD%/lib/commons-lang-2.0.jar +wrapper.java.classpath.3=%PWD%/lib/commons-logging-1.1.jar +wrapper.java.classpath.4=%PWD%/lib/ibatis-2.3.0.677.jar +wrapper.java.classpath.5=%PWD%/lib/log4j-1.2.8.jar +wrapper.java.classpath.6=%PWD%/lib/lucene-core-2.3.2.jar +wrapper.java.classpath.7=%PWD%/lib/ojdbc14-1.10.jar +wrapper.java.classpath.8=%PWD%/lib/picocontainer-1.3.jar +wrapper.java.classpath.9=%PWD%/lib/quartz-1.5.1.jar +wrapper.java.classpath.10=%PWD%/lib/redstone-xmlrpc-1.1_20071120.jar +wrapper.java.classpath.11=%PWD%/lib/redstone-xmlrpc-client-1.1_20071120.jar +wrapper.java.classpath.12=%PWD%/lib/simple-core-3.1.3.jar +wrapper.java.classpath.13=/usr/share/java/tanukiwrapper.jar +wrapper.java.classpath.14=%PWD%/lib/logdriver-0.1.jar +wrapper.java.classpath.15=%PWD%/lib/lucene-analyzers-2.3.2.jar +wrapper.java.classpath.16=/usr/share/nutch +wrapper.java.classpath.17=/usr/share/nutch/conf +wrapper.java.classpath.18=%PWD%/lib/nutch-2008-12-01_04-01-21.jar +wrapper.java.classpath.19=%PWD%/lib/hadoop-0.18.1-core.jar +wrapper.java.classpath.20=%PWD%/lib/commons-httpclient-3.0.jar +wrapper.java.classpath.21=%PWD%/lib/oro-2.0.8.jar +wrapper.java.classpath.22=%PWD%/lib/icu4j-3_8_1.jar +wrapper.java.classpath.23=%PWD%/lib/icu4j-charsets-3_8_1.jar +wrapper.java.classpath.24=%PWD%/lib/commons-cli-1.0.jar +wrapper.java.classpath.25=%PWD%/lib/commons-codec-1.2.jar +wrapper.java.classpath.26=%PWD%/lib/tika-0.1-incubating.jar + + +wrapper.java.library.path.1=/usr/lib +wrapper.java.library.path.2=/usr/lib64 +wrapper.console.format=PM +wrapper.console.loglevel=INFO +wrapper.logfile=%PWD%/rhn_search_daemon.log +wrapper.logfile.format=LPTM +wrapper.logfile.loglevel=DEBUG +wrapper.logfile.maxsize=5m +wrapper.logfile.maxfiles=0 +wrapper.syslog.loglevel=DEBUG + +wrapper.java.additional.1=-Dsearch.config.dir=%PWD%/src/config/search +wrapper.java.additional.2=-Xdebug +wrapper.java.additional.3=-Xnoagent +wrapper.java.additional.4=-Xrunjdwp:transport=dt_socket,address=9999,server=y,suspend=n diff --git a/search-server/spacewalk-search/run.sh b/search-server/spacewalk-search/run.sh new file mode 100755 index 0000000..0bbf58a --- /dev/null +++ b/search-server/spacewalk-search/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +rpm -q tanukiwrapper > /dev/null +RETURN_VAL=$? +if [ ! "$RETURN_VAL" -eq "0" ]; then + echo "Please install tanukiwrapper: up2date tanukiwrapper or yum install tanukiwrapper" + exit 1; +fi + +if [ ! -r "/etc/rhn/search/rhn_search.conf" ]; then + echo "you need to symlink rhn_search.conf as /etc/rhn/search/rhn_search.conf" + exit 2; +fi + +echo "starting server" +export CLASSPATH=`pwd`/build +#java -Djava.library.path=/usr/lib -classpath `build-classpath-directory lib dist` com.redhat.satellite.search.Main +/usr/sbin/tanukiwrapper `pwd`/rhn_search_daemon_dev.conf wrapper.pidfile=`pwd`/rhn-search.pid wrapper.daemonize=FALSE diff --git a/search-server/spacewalk-search/scripts/Reindex.groovy b/search-server/spacewalk-search/scripts/Reindex.groovy new file mode 100644 index 0000000..49e94e0 --- /dev/null +++ b/search-server/spacewalk-search/scripts/Reindex.groovy @@ -0,0 +1,43 @@ +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.WriteQuery; +import com.redhat.satellite.search.config.Configuration; + +def deleteQuery(config, queryName) { + DatabaseManager databaseManager = new DatabaseManager(config) + WriteQuery query = null; + try { + query = databaseManager.getWriterQuery(queryName); + query.delete(null); + } + finally { + query.close(); + } +} + +/** + Currently, only handles Errata and Packages + Need to add: User, System, Docs + **/ +Configuration config = new Configuration() +deleteQueries = ["deleteLastErrata", "deleteLastPackage", + "deleteLastServer", "deleteLastHardwareDevice", + "deleteLastSnapshotTag", "deleteLastServerCustomInfo"] +deleteQueries.each{deleteQuery(config, it)} + +println "Database has been prepared so we can re-index." + +indexWorkDir = config.getString("search.index_work_dir", null); +println "Previous indexes will be deleted on filesystem under ${indexWorkDir}" + +dirs = [indexWorkDir+"/package", indexWorkDir+"/errata", + indexWorkDir+"/server", indexWorkDir+"/hwdevice", + indexWorkDir+"/snapshotTag", indexWorkDir+"/serverCustomInfo"] +dirs.each { + cmd = "rm -r $it" + def proc = cmd.execute() + proc.waitFor() +} + + + + diff --git a/search-server/spacewalk-search/scripts/build.properties b/search-server/spacewalk-search/scripts/build.properties new file mode 100644 index 0000000..6dcc9ac --- /dev/null +++ b/search-server/spacewalk-search/scripts/build.properties @@ -0,0 +1,22 @@ +java.src.dir=src/java +config.src.dir=src/config +java.lib.dir=lib +java.task.lib.dir=../task-lib +report.dir=report +build.dir=build +build.instrument.dir=instrumented +dist.dir=dist +jar.name=spacewalk-search-scripts +jar.version=1.0 +jar.file.name=${jar.name}-${jar.version}.jar +ivyserver= http://jmrodri.fedorapeople.org +ivy.log.module.when.found=false +ivy.log.resolved.revision=false +# Checkstyle config entries +checkstyle.cache.file=.checkstyle_cache +checkstyle.header.file=./buildconf/LICENSE.txt +javadoc.method.scope=public +javadoc.type.scope=package +javadoc.var.scope=package +javadoc.lazy=true +jpackage.jars="commons-lang commons-logging redstone-xmlrpc redstone-xmlrpc-client quartz tanukiwrapper log4j commons-httpclient oro commons-cli commons-codec" diff --git a/search-server/spacewalk-search/scripts/build.xml b/search-server/spacewalk-search/scripts/build.xml new file mode 100644 index 0000000..f45e670 --- /dev/null +++ b/search-server/spacewalk-search/scripts/build.xml @@ -0,0 +1,256 @@ +<project name="search-server" basedir="." default="all"> + <import file="./buildconf/build-utils.xml" /> + + <property file="build.properties" /> + + <path id="project.taskjars"> + <fileset dir="${java.task.lib.dir}" includes="**/*.jar" /> + </path> + + <path id="project.classpath"> + <fileset dir="${java.lib.dir}" includes="**/*.jar" /> + <fileset dir="${java.task.lib.dir}" > + <include name="junit*.jar" /> + <include name="emma*.jar" /> + </fileset> + </path> + + <path id="test.classpath"> + <pathelement location="${build.dir}" /> + <path refid="project.classpath" /> + <pathelement location="${basedir}/src/config" /> + </path> + + <path id="run.classpath"> + <path refid="project.classpath" /> + <pathelement location="${dist.dir}/${jar.file.name}"/> + </path> + + <target name="init-tasks"> + <taskdef name="ivy-retrieve" + classname="fr.jayasoft.ivy.ant.IvyRetrieve" + classpathref="project.taskjars"/> + <taskdef name="ivy-configure" + classname="fr.jayasoft.ivy.ant.IvyConfigure" + classpathref="project.taskjars"/> + <taskdef name="ivy-resolve" + classname="fr.jayasoft.ivy.ant.IvyResolve" + classpathref="project.taskjars"/> + <taskdef name="ivy-publish" + classname="fr.jayasoft.ivy.ant.IvyPublish" + classpathref="project.taskjars"/> + <taskdef resource="checkstyletask.properties"> + <classpath> + <pathelement path="${java.task.lib.dir}/checkstyle-4.1.jar" /> + <pathelement path="${java.task.lib.dir}/commons-beanutils-1.7.0.jar" /> + <pathelement path="${java.task.lib.dir}/commons-logging-1.0.4.jar" /> + <pathelement path="${java.task.lib.dir}/antlr-2.7.6.jar" /> + <pathelement path="${java.task.lib.dir}/regexp-1.3.jar" /> + </classpath> + </taskdef> + <taskdef resource="emma_ant.properties" classpathref="project.taskjars" /> + <taskdef resource="net/sf/antcontrib/antlib.xml"> + <classpath> + <pathelement location="${java.task.lib.dir}/ant-contrib-1.0.jar"/> + </classpath> + </taskdef> + </target> + + <target name="check-testcase-name" if="testcase" depends="init-tasks"> + <propertyregex property="valid.testcase" + input="${testcase}" + regexp="(.*)Test$" + select="\1" + casesensitive="true" /> + </target> + + <target name="check-testcase" depends="check-testcase-name" if="testcase" + unless="valid.testcase"> + <fail message="Invalid testcase name: ${testcase}. Class name must look like .*Test"/> + </target> + + <target name="test" depends="compile, check-testcase" + description="Run the tests (use -Dtestcase=<testcase>)"> + <mkdir dir="${report.dir}" /> + <!-- set testcase to wildcard "*" to test everything, unless + a specific testcase name was supplied. --> + <if> + <not> + <isset property="${testcase}" /> + </not> + <then> + <property name="testcase" value="*Test"/> + </then> + </if> + <junit> + <sysproperty key="rhn.config.dir" value="/etc/rhn" /> + <sysproperty key="search.config.dir" value="src/config/search" /> + <sysproperty key="log4j.configuration" value="log4j.test.properties" /> + <classpath refid="test.classpath" /> + <formatter type="brief" usefile="false" /> + <formatter type="xml" /> + <batchtest fork="yes" todir="${report.dir}"> + <fileset dir="${build.dir}" includes="**/${testcase}.class" /> + </batchtest> + </junit> + </target> + + <target name="checkstyle" depends="init-tasks, package"> + <delete dir="${report.dir}" /> + <mkdir dir="${report.dir}" /> + <checkstyle config="checkstyle.xml" failOnViolation="true" > + <property key="javadoc.method.scope" value="public" /> + <property key="javadoc.type.scope" value="package" /> + <property key="javadoc.var.scope" value="package" /> + <property key="javadoc.lazy" value="${javadoc.lazy}" /> + <fileset dir="${java.src.dir}" > + <include name="**/*.java" /> + <exclude name="**/tests/**" /> + </fileset> + <classpath> + <fileset dir="${java.lib.dir}" includes="**/*.jar" /> + <fileset dir="${java.task.lib.dir}" includes="junit*.jar" /> + <fileset dir="${dist.dir}" includes="${jar.file.name}" /> + </classpath> + <formatter type="plain" /> + <formatter type="xml" toFile="${report.dir}/checkstyle_report.xml" /> + </checkstyle> + </target> + + <target name="resolve-local" description="resolve jars via jpackage" if="installbuild"> + <copy toDir="${java.lib.dir}" overwrite="true"> + <fileset dir="buildconf/tempjars"> + <include name="**/*.jar" /> + </fileset> + </copy> + + <jpackage-deps jars="${jpackage.jars}" dir="${java.lib.dir}" /> + </target> + + <target name="resolve-ivy" description="retrieve dependencies with ivy" + depends="init-tasks" unless="installbuild"> + <!-- properties set here to show we are overriding default ivy values --> + <property name="ivy.default.ivy.user.dir" value="${user.home}/.ivy/search-server"/> + <property name="ivy.local.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> + <property name="ivy.ivyrep.default.ivy.root" value="${ivyserver}/ivy/" /> + <property name="ivy.ivyrep.default.artifact.root" value="${ivyserver}/ivy/" /> + <property name="ivy.ivyrep.default.artifact.pattern" value="[artifact]-[revision].[ext]" /> + <ivy-retrieve/> + </target> + + <target name="resolve" depends="resolve-ivy,resolve-local" /> + + <target name="prep"> + <mkdir dir="${build.dir}" /> + <mkdir dir="${dist.dir}" /> + <mkdir dir="${java.lib.dir}" /> + </target> + + <target name="clean"> + <delete dir="${build.dir}" /> + <delete dir="${build.instrument.dir}" /> + <delete dir="${dist.dir}" /> + <delete dir="${java.lib.dir}" /> + <delete dir="${report.dir}" /> + <delete file="coverage.ec" /> + <delete file="rhn_search_daemon.log" /> + </target> + + <target name="compile" depends="prep, resolve"> + <javac srcdir="${java.src.dir}" + classpathref="project.classpath" + destdir="${build.dir}" + deprecation="on" + excludesfile="buildconf/exclude" + debug="on" /> + <copy file="${config.src.dir}/log4j.properties" todir="${build.dir}" /> + </target> + + <target name="package" depends="compile"> + <jar basedir="${build.dir}" + destfile="${dist.dir}/${jar.file.name}" + excludes="**/tests/**" + includes="**/**" /> + </target> + + <target name="create-local-repo" description="Create a local Ivy repo"> + <mkdir dir="${user.home}/.ivy/search-server/local/" /> + </target> + + <target name="clean-cache" description="Removes the Ivy jar cache"> + <delete dir="${lib.dir}" quiet="true" /> + <delete dir="${user.home}/.ivy/search-server/cache/" quiet="true" /> + </target> + + <target name="all" depends="package" /> + + <target name="init-install"> + <property name="installbuild" value="true"/> + </target> + + <target name="install" depends="init-install,package" /> + + <target name="instrument" depends="init-tasks"> + <mkdir dir="${report.dir}" /> + <mkdir dir="${build.instrument.dir}" /> + + <emma enabled="true"> + <instr merge="no" + mode="overwrite" + instrpath="${build.dir}" + outfile="${report.dir}/coverage.em" > + <filter excludes="*Exception" /> + <filter excludes="*.tests.*" /> + </instr> + <!-- + <instr outdir="${build.instrument.dir}" merge="no" + instrpath="${build.dir}" + mode="fullcopy" + outfile="${report.dir}/coverage.em" > + <filter excludes="*Exception" /> + <filter excludes="*.tests.*" /> + </instr> + --> + </emma> + + <!-- + <copy toDir="${build.instrument.dir}/classes"> + <fileset dir="${build.dir}"> + <include name="**/*.xml" /> + </fileset> + </copy> + --> + </target> + + <target name="coverage-report" depends="instrument"> + <emma enabled="true"> + <report sourcepath="${java.src.dir}" + sort="+block,+name,+method,+class" + metrics="method:70,block:80,line:80,class:100"> + <fileset dir="${report.dir}" includes="*.em" /> + <fileset dir="." includes="*.ec" /> + <html outfile="${report.dir}/emma.html" + depth="method" + columns="name,class,method,block,line" /> + <xml outfile="${report.dir}/emma.xml" + depth="method" + columns="name,class,method,block,line" /> + </report> + </emma> + </target> + + + <target name="indexDocs" depends="package" + description="Crawls help docs with nutch and generates a lucene index"> + <java + classname="com.redhat.satellite.search.index.docs.WebCrawl" + dir="${basedir}" + fork="true" + failonerror="true"> + + <arg value="-h"/> + <classpath refid="run.classpath" /> + </java> + </target> + +</project> diff --git a/search-server/spacewalk-search/scripts/buildconf/LICENSE.txt b/search-server/spacewalk-search/scripts/buildconf/LICENSE.txt new file mode 100644 index 0000000..629462a --- /dev/null +++ b/search-server/spacewalk-search/scripts/buildconf/LICENSE.txt @@ -0,0 +1,14 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ diff --git a/search-server/spacewalk-search/scripts/buildconf/build-utils.xml b/search-server/spacewalk-search/scripts/buildconf/build-utils.xml new file mode 100644 index 0000000..d428618 --- /dev/null +++ b/search-server/spacewalk-search/scripts/buildconf/build-utils.xml @@ -0,0 +1,19 @@ +<project name="build-utils.xml"> + <!-- + creates symlinks for jpackage jars + requires: jpackage-utils + other: requires rpms of the jars specified to be installed + --> + <macrodef name="jpackage-deps"> + <attribute name="jars" /> + <attribute name="dir" /> + + <sequential> + <exec executable="build-jar-repository" failonerror="true" > + <arg value="--preserve-naming" /> + <arg value="-s @{dir}" /> + <arg line="@{jars}" /> + </exec> + </sequential> + </macrodef> +</project> diff --git a/search-server/spacewalk-search/scripts/buildconf/exclude b/search-server/spacewalk-search/scripts/buildconf/exclude new file mode 100644 index 0000000..e69de29 diff --git a/search-server/spacewalk-search/scripts/checkstyle.xml b/search-server/spacewalk-search/scripts/checkstyle.xml new file mode 100644 index 0000000..2cb51d5 --- /dev/null +++ b/search-server/spacewalk-search/scripts/checkstyle.xml @@ -0,0 +1,233 @@ +<?xml version="1.0"?> +<!DOCTYPE module PUBLIC + "-//Puppy Crawl//DTD Check Configuration 1.1//EN" + "http://www.puppycrawl.com/dtds/configuration_1_1.dtd"> + +<!-- + + Checkstyle configuration that checks the sun coding conventions from: + + - the Java Language Specification at + http://java.sun.com/docs/books/jls/second_edition/html/index.html + + - the Sun Code Conventions at http://java.sun.com/docs/codeconv/ + + - the Javadoc guidelines at + http://java.sun.com/j2se/javadoc/writingdoccomments/index.html + + - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html + + - some best practices + + Checkstyle is very configurable. Be sure to read the documentation at + http://checkstyle.sf.net (or in your downloaded distribution). + + Most Checks are configurable, be sure to consult the documentation. + + To completely disable a check, just comment it out or delete it from the file. + + Finally, it is worth reading the documentation. + +--> + +<module name="Checker"> + + <!-- Checks that a package.html file exists for each package. --> + <!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml --> +<!-- <module name="PackageHtml"/> --> + + <!-- Checks whether files end with a new line. --> + <!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile --> + <module name="NewlineAtEndOfFile"/> + + <!-- Checks that property files contain the same keys. --> + <!-- See http://checkstyle.sf.net/config_misc.html#Translation --> + <module name="Translation"/> + + <module name="TreeWalker"> + + <property name="cacheFile" value="${checkstyle.cache.file}"/> + + <!-- ************************************************************** --> + <!-- Checks that are different from the sun coding conventions ones --> + <!-- ************************************************************** --> + <property name="tabWidth" value="4"/> + <module name="LeftCurly" /> + <module name="RightCurly"> + <property name="option" value="alone"/> + </module> + <module name="LineLength"> + <property name="ignorePattern" value="import"/> + <property name="max" value="92"/> + </module> + + <!-- ************************************************************** --> + <!-- Default Sun coding conventions checks --> + <!-- ************************************************************** --> + + <!-- Checks for Javadoc comments. --> + <!-- See http://checkstyle.sf.net/config_javadoc.html --> + <module name="JavadocMethod"> + <property name="scope" value="${javadoc.method.scope}" /> + <property name="allowUndeclaredRTE" value="true" /> + <property name="allowThrowsTagsForSubclasses" value="${javadoc.lazy}" /> + <property name="allowMissingParamTags" value="${javadoc.lazy}" /> + <property name="allowMissingThrowsTags" value="${javadoc.lazy}" /> + <property name="allowMissingReturnTag" value="${javadoc.lazy}" /> + </module> + <module name="JavadocType"> + <property name="versionFormat" value="\$Rev.*\$"/> + <property name="scope" value="${javadoc.type.scope}" /> + </module> + <module name="JavadocVariable"> + <property name="scope" value="${javadoc.var.scope}" /> + <property name="severity" value="ignore"/> + </module> + +<!-- JavadocStyle not supported by some versions. --> +<!-- <module name="JavadocStyle"/> --> + + + <!-- Checks for Naming Conventions. --> + <!-- See http://checkstyle.sf.net/config_naming.html --> + <module name="ConstantName"/> + <module name="LocalFinalVariableName"/> + <module name="LocalVariableName"/> + <module name="MethodName"/> + <module name="PackageName"> + <property name="format" value="^[a-z]+(\.[A-Za-z][A-Za-z0-9]*)*$"/> + </module> + <module name="ParameterName"/> + <module name="StaticVariableName"/> + <module name="TypeName"/> + + + <!-- Checks for Headers --> + <!-- See http://checkstyle.sf.net/config_header.html --> + <module name="Header"> + <!-- The follow property value demonstrates the ability --> + <!-- to have access to ANT properties. In this case it uses --> + <!-- the ${basedir} property to allow Checkstyle to be run --> + <!-- from any directory within a project. --> + <property name="headerFile" value="${checkstyle.header.file}"/> + </module> + + <!-- Following interprets the header file as regular expressions. --> + <!-- <module name="RegexpHeader"/> --> + + + + <!-- Checks for imports --> + <!-- See http://checkstyle.sf.net/config_import.html --> + <module name="AvoidStarImport"/> + <module name="IllegalImport"/> <!-- defaults to sun.* packages --> + <module name="RedundantImport"/> + <module name="UnusedImports"/> + + + <!-- Checks for Size Violations. --> + <!-- See http://checkstyle.sf.net/config_sizes.html --> + <module name="MethodLength"> + <property name="tokens" value="METHOD_DEF"/> + <property name="max" value="150"/> + <property name="countEmpty" value="false"/> + </module> + <module name="ParameterNumber"/> + + + <!-- Checks for whitespace --> + <!-- See http://checkstyle.sf.net/config_whitespace.html --> + <module name="EmptyForIteratorPad"/> + <module name="NoWhitespaceAfter"> + <property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, UNARY_PLUS, TYPECAST, GENERIC_START"/> + </module> + <module name="NoWhitespaceBefore"> + <property name="tokens" value="SEMI, POST_DEC, POST_INC, GENERIC_START, GENERIC_END"/> + </module> + <module name="OperatorWrap"> + <property name="option" value="eol" /> + </module> + <module name="ParenPad"/> +<!-- <module name="TypecastParenPad"/> --> + <module name="TabCharacter"/> + <module name="WhitespaceAfter"> + <property name="tokens" value="COMMA, SEMI, GENERIC_END"/> + </module> + <module name="WhitespaceAround"> + <property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, +COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LCURLY, LE, LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, +LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS, +MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION, RCURLY, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN, STAR, STAR_ASSIGN, TYPE_EXTENSION_AND, WILDCARD_TYPE"/> + </module> + <module name="MethodParamPad"> + <property name="allowLineBreaks" value="true"/> + </module> + + + <!-- Modifier Checks --> + <!-- See http://checkstyle.sf.net/config_modifiers.html --> + <module name="ModifierOrder"/> + <module name="RedundantModifier"/> + + + <!-- Checks for blocks. You know, those {}'s --> + <!-- See http://checkstyle.sf.net/config_blocks.html --> + <module name="AvoidNestedBlocks"/> + <module name="EmptyBlock"> + <property name="option" value="text"/> + </module> + + <module name="NeedBraces"/> + + + <!-- Checks for common coding problems --> + <!-- See http://checkstyle.sf.net/config_coding.html --> + <!-- <module name="AvoidInlineConditionals"/> --> +<!-- <module name="CovariantEquals"/> --> + <module name="DoubleCheckedLocking"/> <!-- MY FAVOURITE --> + <module name="EmptyStatement"/> + <module name="EqualsHashCode"/> + <module name="HiddenField"/> + <module name="IllegalInstantiation"/> + <module name="InnerAssignment"/> + <!-- <module name="MagicNumber" /> --> + <module name="MissingSwitchDefault"/> +<!-- <module name="RedundantThrows"/> --> + <module name="SimplifyBooleanExpression"/> + <module name="SimplifyBooleanReturn"/> +<!-- <module name="IllegalCatch"/> --> +<!-- <module name="PackageDeclaration"/> --> + + <!-- Checks for class design --> + <!-- See http://checkstyle.sf.net/config_design.html --> +<!-- <module name="DesignForExtension"/>--> +<!-- <module name="FinalClass"/> --> + <module name="HideUtilityClassConstructor"/> + <module name="InterfaceIsType"/> + <module name="VisibilityModifier"> + <property name="protectedAllowed" value="true" /> + </module> + + + <!-- Miscellaneous other checks. --> + <!-- See http://checkstyle.sf.net/config_misc.html --> + <module name="ArrayTypeStyle"/> +<!-- <module name="FinalParameters"/> --> +<!-- + <module name="GenericIllegalRegexp"> + <property name="format" value="System\.out\.println"/> + </module> +--> + <module name="UpperEll"/> + + <!-- Optional Checks --> + <!-- (gack, not available until version 3 of checkstyle --> +<!-- + <module name="usage.OneMethodPrivateField"/> + <module name="usage.UnusedLocalVariable"/> + <module name="usage.UnusedParameter"/> + <module name="usage.UnusedPrivateField"/> +--> + </module> + +</module> diff --git a/search-server/spacewalk-search/scripts/ivy.xml b/search-server/spacewalk-search/scripts/ivy.xml new file mode 100644 index 0000000..cffd1c1 --- /dev/null +++ b/search-server/spacewalk-search/scripts/ivy.xml @@ -0,0 +1,28 @@ +<ivy-module version="1.0"> + <info organisation="redhat" module="rhn-java" /> + <dependencies> + <dependency org="redhat" name="simple-core" rev="3.1.3" /> + <dependency org="redhat" name="lucene-core" rev="2.3.2" /> + <dependency org="redhat" name="lucene-analyzers" rev="2.3.2" /> + <dependency org="redhat" name="commons-lang" rev="2.0" /> + <dependency org="redhat" name="commons-logging" rev="1.1" /> + <dependency org="redhat" name="redstone-xmlrpc" rev="1.1_20071120" /> + <dependency org="redhat" name="redstone-xmlrpc-client" rev="1.1_20071120" /> + <dependency org="redhat" name="ibatis" rev="2.3.0.677" /> + <dependency org="redhat" name="ojdbc14" rev="1.10" /> + <dependency org="redhat" name="quartz" rev="1.5.1" /> + <dependency org="redhat" name="picocontainer" rev="1.3" /> + <dependency org="redhat" name="tanukiwrapper" rev="3.1.2" /> + <dependency org="redhat" name="log4j" rev="1.2.8" /> + <dependency org="redhat" name="logdriver" rev="0.1" /> + <dependency org="redhat" name="nutch" rev="1.0dev_2008_05_14" /> + <dependency org="redhat" name="hadoop" rev="0.16.4-core" /> + <dependency org="redhat" name="commons-cli" rev="1.0" /> + <!-- dependency org="redhat" name="commons-httpclient" rev="3.0" / --> + <!-- dependency org="redhat" name="oro" rev="2.0.8" / --> + <!-- dependency org="redhat" name="commons-codec" rev="1.2" / --> + <!-- dependency org="redhat" name="tika" rev="0.1-incubating" / --> + <!-- dependency org="redhat" name="icu4j" rev="3_8_1" / --> + <!-- dependency org="redhat" name="icu4j-charsets" rev="3_8_1" / --> + </dependencies> +</ivy-module> diff --git a/search-server/spacewalk-search/scripts/lukeall-0.8.1.jar b/search-server/spacewalk-search/scripts/lukeall-0.8.1.jar new file mode 100644 index 0000000..347aee6 Binary files /dev/null and b/search-server/spacewalk-search/scripts/lukeall-0.8.1.jar differ diff --git a/search-server/spacewalk-search/scripts/maketar.sh b/search-server/spacewalk-search/scripts/maketar.sh new file mode 100755 index 0000000..6a389be --- /dev/null +++ b/search-server/spacewalk-search/scripts/maketar.sh @@ -0,0 +1,30 @@ +if [ -a version ]; then + version=`cat version | awk '/^[0-9].[0-9].[0-9]* / { print $1 }'` + + if [ "$version" == "" ]; then + echo "Version not found" + exit -1 + fi + + echo "Creating /tmp/rhn-search-$version" + mkdir /tmp/rhn-search-$version + + echo "Copying repo to /tmp/rhn-search-$version/" + cp -R * /tmp/rhn-search-$version/ + + cd /tmp/ + rm -f rhn-search-$version/scripts/maketar.sh + echo "Creating /tmp/rhn-search-$version.tar.gz file" + tar -czf /tmp/rhn-search-$version.tar.gz rhn-search-$version/ --exclude .svn + cd - + + echo "Removing /tmp/rhn-search-$version" + rm -rf /tmp/rhn-search-$version + + echo "" + echo "You can find your tar at /tmp/rhn-search-$version.tar.gz" + echo "" + +else + echo "You moron, you gotta run this from the directory containing the version file." +fi diff --git a/search-server/spacewalk-search/scripts/reindex.sh b/search-server/spacewalk-search/scripts/reindex.sh new file mode 100755 index 0000000..cc7b312 --- /dev/null +++ b/search-server/spacewalk-search/scripts/reindex.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +RHN_SEARCH_HOME="../" + +for f in $RHN_SEARCH_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done +for f in $RHN_SEARCH_HOME/dist/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done +export CLASSPATH + +groovy Reindex.groovy + + diff --git a/search-server/spacewalk-search/scripts/search.admin.updateIndex.test.py b/search-server/spacewalk-search/scripts/search.admin.updateIndex.test.py new file mode 100755 index 0000000..8962c9d --- /dev/null +++ b/search-server/spacewalk-search/scripts/search.admin.updateIndex.test.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import sys +import xmlrpclib + +if __name__ == "__main__": + if len(sys.argv) < 2: + print "Usage: %s INDEX_NAME" % (sys.argv[0]) + sys.exit(1) + port = 2828 + addr = "http://127.0.0.1:%s" % (port) + client = xmlrpclib.ServerProxy(addr) + indexName = sys.argv[1] + result = client.admin.updateIndex(indexName) + print "Return from admin.updateIndex(%s) = %s" % (indexName, result) diff --git a/search-server/spacewalk-search/scripts/search.py b/search-server/spacewalk-search/scripts/search.py new file mode 100755 index 0000000..c8d0a04 --- /dev/null +++ b/search-server/spacewalk-search/scripts/search.py @@ -0,0 +1,69 @@ +#!/usr/bin/python + +import xmlrpclib +from optparse import OptionParser + +indexName = "package" +usage = "usage: %prog [options] search term" +desc = "%prog searches for package (default) or systems with the given \ +search criteria" + +parser = OptionParser(usage=usage, description=desc) +parser.add_option("--sessionid", dest="sessionid", type="int", help="PXT sessionid") +parser.add_option("--package", action="store_true", dest="package", + help="search packages", default=True) +parser.add_option("--system", action="store_true", dest="system", + help="search systems", default=False) +parser.add_option("--indexName", dest="indexName", type="string", + help="lucene index name to search ex: package server hwdevice snapshotTag errata") +parser.add_option("--serverAddr", dest="serverAddr", type="string", default="localhost", + help="Server to authenticate to, NOT WHERE SEARCH SERVER RUNS") +parser.add_option("--username", dest="username", type="string", help="username") +parser.add_option("--password", dest="password", type="string", help="password") +parser.add_option("--debug", action="store_true", dest="debug", default=False, + help="enable debug output") + +(options, terms) = parser.parse_args() +if len(terms) < 1: + parser.error("please supply a search term\n" + str(parser.print_help())) + +if not options.sessionid and (not options.username or not options.password): + print parser.print_help() + parser.exit() + + +if options.package: + indexName = "package" + +if options.system: + indexName = "server" + +if options.indexName: + indexName = options.indexName + + +sessionid = None +if options.sessionid: + sessionid = options.sessionid + print "Using passed in authentication info, sessionid = %s" % (sessionid) +else: + xmlrpcURL = "http://%s/rhn/rpc/api" % (options.serverAddr) + print "Getting authentication information from: %s" % (xmlrpcURL) + rhnclient = xmlrpclib.Server(xmlrpcURL) + authSessionId = rhnclient.auth.login(options.username, options.password) + sessionid = int(authSessionId.split('x')[0]) + +url = "http://localhost:2828/RPC2" +print "Connecting to SearchServer: (%s)" % url +client = xmlrpclib.Server(url, verbose=options.debug) + +term = " ".join(terms) +print "searching for (%s) matching criteria: (%s)" % (indexName, str(term)) +items = client.index.search(sessionid, indexName, term) + +print "We got (%d) items back." % len(items) +print items + +#Remember to logout if the user didn't supply the sessionid +if not options.sessionid: + rhnclient.auth.logout(authSessionId) diff --git a/search-server/spacewalk-search/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java b/search-server/spacewalk-search/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java new file mode 100644 index 0000000..7a9f671 --- /dev/null +++ b/search-server/spacewalk-search/scripts/src/java/com/redhat/satellite/search/index/docs/WebCrawl.java @@ -0,0 +1,304 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.docs; + +import org.apache.nutch.crawl.Injector; +import org.apache.nutch.crawl.Generator; +import org.apache.nutch.crawl.CrawlDb; +import org.apache.nutch.crawl.LinkDb; +import org.apache.nutch.fetcher.Fetcher; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.nutch.parse.ParseSegment; +import org.apache.nutch.indexer.DeleteDuplicates; +import org.apache.nutch.indexer.IndexMerger; +import org.apache.nutch.indexer.Indexer; +import org.apache.nutch.util.HadoopFSUtil; +import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.NutchJob; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; + +import org.apache.commons.cli.ParseException; + +import org.apache.log4j.Logger; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.io.IOException; + + +/** +Crawls a set of urls and indexes their content. +Based off of nutch's crawl + * + * @version $Rev: $ + */ +public class WebCrawl { + private static Logger log = Logger.getLogger(WebCrawl.class); + + private Configuration conf; + private JobConf job; + private String inputUrlFile; + private String tmpCrawlDir; + private String outputIndexDir; + + private int depth; + private int threads; + private long topN; + + /** + * Constructor + */ + public WebCrawl() { + conf = NutchConfiguration.create(); + conf.addResource("crawl-tool.xml"); + job = new NutchJob(conf); + threads = job.getInt("fetcher.threads.fetch", 10); + depth = 5; + topN = Long.MAX_VALUE; + } + + /** + * @return Returns the dir used to store tmp crawl data + */ + public String getTmpCrawlDir() { + return tmpCrawlDir; + } + /** + * @param tmpCrawlDirIn the dir used to store tmp crawl data + */ + public void setTmpCrawlDir(String tmpCrawlDirIn) { + tmpCrawlDir = tmpCrawlDirIn; + } + /** + * @return Returns the file used to seed the url crawl + */ + public String getInputUrlFile() { + return inputUrlFile; + } + /** + * @param inputUrlFileIn file used to seed the url crawl + */ + public void setInputUrlFile(String inputUrlFileIn) { + inputUrlFile = inputUrlFileIn; + } + /** + * @return Returns the dir to store the index of the crawled docs + */ + public String getOutputIndexDir() { + return outputIndexDir; + } + /** + * @param outputIndexDirIn the dir to store the index of the crawled docs + */ + public void setOutputIndexDir(String outputIndexDirIn) { + outputIndexDir = outputIndexDirIn; + } + /** + * @return Returns the maximum depth of recursion allowed + */ + public int getDepth() { + return depth; + } + /** + * @param depthIn the maximum depth of recursion allowed + */ + public void setDepth(int depthIn) { + depth = depthIn; + } + /** + * @return Returns the maximum number of fetcher threads used + */ + public int getThreads() { + return threads; + } + /** + * @param threadsIn the maximum number of fetcher threads used + */ + public void setThreads(int threadsIn) { + threads = threadsIn; + } + /** + * @return Returns the maximum number of "out links" to follow in any given page + */ + public long getTopN() { + return topN; + } + /** + * @param topNIn the maximum number of "out links" to follow in any given page + */ + public void setTopN(long topNIn) { + topN = topNIn; + } + + protected String getDate() { + return new SimpleDateFormat("yyyyMMddHHmmss").format( + new Date(System.currentTimeMillis())); + } + + /** + * Uses nutch to crawl a list of ulrs defined in "inputUrlFile" + * Temporary scratch storage is held at "tmpCrawlDir" + * The desired output is an index of crawled pages, stored at "outputIndexDir" + * + * @return true when urls successfully crawled/indexed + */ + @SuppressWarnings("deprecation") + public boolean crawl() throws IOException { + log.info("Performing crawl with following config options: "); + log.info("inputUrlFile = " + inputUrlFile); + log.info("tmpCrawlDir = " + tmpCrawlDir); + log.info("outputIndexDir = " + outputIndexDir); + log.info("threads = " + threads); + log.info("depth = " + depth); + log.info("topN = " + topN); + + Path inputPath = new Path(inputUrlFile); + Path linkDb = new Path(tmpCrawlDir + "/linkdb"); + Path segments = new Path(tmpCrawlDir + "/segments"); + Path indexes = new Path(tmpCrawlDir + "/indexes"); + Path index = new Path(outputIndexDir); + Path crawlDb = new Path(tmpCrawlDir + "/crawldb"); + + Path tmpDir = job.getLocalPath("crawl" + Path.SEPARATOR + getDate()); + Injector injector = new Injector(conf); + Generator generator = new Generator(conf); + Fetcher fetcher = new Fetcher(conf); + ParseSegment parseSegment = new ParseSegment(conf); + CrawlDb crawlDbTool = new CrawlDb(conf); + LinkDb linkDbTool = new LinkDb(conf); + Indexer indexer = new Indexer(conf); + DeleteDuplicates dedup = new DeleteDuplicates(conf); + IndexMerger merger = new IndexMerger(conf); + + FileSystem fs = FileSystem.get(job); + + log.info("Create a new database (crawlDB) of link information"); + injector.inject(crawlDb, inputPath); + + for (int i = 0; i < depth; i++) { + log.info("Generate a fetch list from info in the crawlDB."); + Path segment = generator.generate(crawlDb, segments, -1, topN, + System.currentTimeMillis()); + if (segment == null) { + log.info("Stopping at depth = " + i + " instead of " + (depth - 1) + + " - no more URLs to fetch."); + break; + } + log.info("Fetch links"); + fetcher.fetch(segment, threads); // fetch it + if (!Fetcher.isParsing(job)) { + log.info("Parsing Segment"); + parseSegment.parse(segment); // parse it, if needed + } + Path[] p = new Path[1]; + p[0] = segment; + log.info("Update CrawlDB"); + crawlDbTool.update(crawlDb, p, true, true); // update crawldb + } + + linkDbTool.invert(linkDb, segments, true, true, false); // invert links + + // Delete old indexes + if (fs.exists(indexes)) { + log.info("Deleting old indexes: $indexes"); + fs.delete(indexes); + } + + // Delete old index + if (fs.exists(index)) { + log.info("Deleting old merged index: $index"); + fs.delete(index); + } + + // index, dedup & merge + indexer.index(indexes, crawlDb, linkDb, + fs.listPaths(segments, HadoopFSUtil.getPassAllFilter())); + + Path[] p = new Path[1]; + p[0] = indexes; + dedup.dedup(p); + merger.merge(fs.listPaths(indexes, HadoopFSUtil.getPassAllFilter()), + index, tmpDir); + log.info("Crawl finished"); + return true; + } + + + /** + * Performs a web crawl: + * @param args command line arguments + */ + public static void main(String[] args) throws IOException { + + WebCrawl wCrawl = new WebCrawl(); + + Options options = new Options(); + options.addOption("i", "inputUrlFile", true, + "file holding 'urls' file to seed web page crawling"); + options.addOption("o", "outputDir", true, "temp crawl output dir"); + options.addOption("x", "docsIndexDir", true, "docs index output dir"); + options.addOption("t", "threads", true, "number of threads"); + options.addOption("d", "depth", true, "depth to recurse towards"); + options.addOption("n", "topN", true, "maximum number of out links to follow"); + options.addOption("h", "help", false, "print help message"); + + CommandLineParser parser = new PosixParser(); + try { + CommandLine line = parser.parse(options, args); + + if (line.hasOption("h")) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("WebCrawl", options); + return; + } + if (line.hasOption("i")) { + wCrawl.setInputUrlFile(line.getOptionValue("i")); + } + if (line.hasOption("x")) { + wCrawl.setOutputIndexDir(line.getOptionValue("x")); + } + if (line.hasOption("o")) { + wCrawl.setTmpCrawlDir(line.getOptionValue("o")); + } + if (line.hasOption("t")) { + wCrawl.setThreads(Integer.parseInt(line.getOptionValue("t"))); + } + if (line.hasOption("d)")) { + wCrawl.setDepth(Integer.parseInt(line.getOptionValue("d"))); + } + if (line.hasOption("n")) { + wCrawl.setTopN(Integer.parseInt(line.getOptionValue("n"))); + } + + } + catch (ParseException exp) { + System.err.println("Parsing failed. Reason: " + exp.getMessage()); + } + + if (!wCrawl.crawl()) { + System.err.println("Error -- WebCrawl.crawl() Failed!"); + } + } + + +} diff --git a/search-server/spacewalk-search/scripts/webCrawl.sh b/search-server/spacewalk-search/scripts/webCrawl.sh new file mode 100755 index 0000000..da352f6 --- /dev/null +++ b/search-server/spacewalk-search/scripts/webCrawl.sh @@ -0,0 +1,17 @@ +#!/bin/sh +SEARCH_HOME=.. +NUTCH_HOME=/usr/share/nutch + +CLASSPATH=${CLASSPATH}:${SEARCH_HOME}/build +#Put our nutch/conf dir before the NUTCH_HOME so ensure our config files are used +CLASSPATH=${CLASSPATH}:${SEARCH_HOME}/nutch/conf +CLASSPATH=${CLASSPATH}:${NUTCH_HOME} +for f in ${SEARCH_HOME}/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +java -cp ${CLASSPATH} com.redhat.satellite.search.scheduler.tasks.crawl.WebCrawl --inputUrlFile ${SEARCH_HOME}/nutch/urls --outputDir ./crawl_output --docsIndexDir ./docsIndexDir --depth 2 --threads 4 + + + + diff --git a/search-server/spacewalk-search/spacewalk-search.spec b/search-server/spacewalk-search/spacewalk-search.spec new file mode 100644 index 0000000..fcc3580 --- /dev/null +++ b/search-server/spacewalk-search/spacewalk-search.spec @@ -0,0 +1,255 @@ +%{!?__redhat_release:%define __redhat_release UNKNOWN} + +Name: spacewalk-search +Summary: Spacewalk Full Text Search Server +Group: Applications/Internet +License: GPLv2 +Version: 0.8.1 +Release: 1%{?dist} +# This src.rpm is cannonical upstream +# You can obtain it using this set of commands +# git clone git://git.fedorahosted.org/git/spacewalk.git/ +# cd search-server +# make test-srpm +URL: https://fedorahosted.org/spacewalk +Source0: %{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildArch: noarch + +#Requires: apache-ibatis-sqlmap +Requires: doc-indexes +Requires: jakarta-commons-cli +Requires: jakarta-commons-codec +Requires: jakarta-commons-httpclient +Requires: jakarta-commons-lang >= 0:2.1 +Requires: jakarta-commons-logging +Requires: jpackage-utils >= 0:1.5 +Requires: log4j +Requires: oro +#Requires: lucene +Requires: quartz +Requires: redstone-xmlrpc +#Requires: picocontainer +Requires: tanukiwrapper +Obsoletes: rhn-search < 5.3.0 +BuildRequires: ant +#BuildRequires: apache-ibatis-sqlmap +BuildRequires: jakarta-commons-cli +BuildRequires: jakarta-commons-codec +BuildRequires: jakarta-commons-httpclient +BuildRequires: jakarta-commons-lang >= 0:2.1 +BuildRequires: jakarta-commons-logging +BuildRequires: java-devel >= 1.6.0 +BuildRequires: log4j +BuildRequires: oro +#BuildRequires: lucene +BuildRequires: quartz +BuildRequires: redstone-xmlrpc +#BuildRequires: picocontainer +BuildRequires: tanukiwrapper +Requires(post): chkconfig +Requires(preun): chkconfig +# This is for /sbin/service +Requires(preun): initscripts + +%description +This package contains the code for the Full Text Search Server for +Spacewalk Server. + +%prep +%setup -n %{name}-%{version} + +%install +rm -fr ${RPM_BUILD_ROOT} +ant -Djar.version=%{version} install +install -d -m 755 $RPM_BUILD_ROOT%{_sysconfdir}/rhn/search +install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search +install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/indexes +install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/lib +install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/classes/com/redhat/satellite/search/db +install -d -m 755 $RPM_BUILD_ROOT%{_initrddir} +install -d -m 755 $RPM_BUILD_ROOT%{_bindir} +install -d -m 755 $RPM_BUILD_ROOT%{_var}/log/rhn/search +install -d -m 755 $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/nutch +install -d -m 755 $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d +install -p -m 644 dist/%{name}-%{version}.jar $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/lib/ +# using install -m does not preserve the symlinks +cp -d lib/* $RPM_BUILD_ROOT/%{_prefix}/share/rhn/search/lib +install -p -m 644 src/config/log4j.properties $RPM_BUILD_ROOT/%{_prefix}/share/rhn/search/classes/log4j.properties +install -p -m 644 src/config/etc/logrotate.d/rhn-search $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d/rhn-search +install -p -m 644 src/config/com/redhat/satellite/search/db/* $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/classes/com/redhat/satellite/search/db +install -p -m 755 src/config/rhn-search $RPM_BUILD_ROOT%{_initrddir} +ln -s -f /usr/sbin/tanukiwrapper $RPM_BUILD_ROOT%{_bindir}/rhnsearchd +install -p -m 644 src/config/search/rhn_search.conf $RPM_BUILD_ROOT%{_sysconfdir}/rhn/search/rhn_search.conf +install -p -m 644 src/config/search/rhn_search_daemon.conf $RPM_BUILD_ROOT%{_sysconfdir}/rhn/search/rhn_search_daemon.conf +ln -s -f %{_prefix}/share/rhn/search/lib/spacewalk-search-%{version}.jar $RPM_BUILD_ROOT%{_prefix}/share/rhn/search/lib/spacewalk-search.jar + +%clean +rm -rf $RPM_BUILD_ROOT + +%post +# This adds the proper /etc/rc*.d links for the script +/sbin/chkconfig --add rhn-search + +%preun +if [ $1 = 0 ] ; then + /sbin/service rhn-search stop >/dev/null 2>&1 + /sbin/chkconfig --del rhn-search +fi + +%files +%defattr(644,root,root,755) +%attr(755, root, root) %{_var}/log/rhn/search +%{_prefix}/share/rhn/search/lib/* +%{_prefix}/share/rhn/search/classes/log4j.properties +%{_prefix}/share/rhn/search/classes/com/* +%attr(755, root, root) %{_prefix}/share/rhn/search/indexes +%attr(755, root, root) %{_initrddir}/rhn-search +%attr(755, root, root) %{_bindir}/rhnsearchd +%dir %{_sysconfdir}/rhn/search/ +%config(noreplace) %{_sysconfdir}/rhn/search/rhn_search.conf +%config(noreplace) %{_sysconfdir}/rhn/search/rhn_search_daemon.conf +%{_sysconfdir}/logrotate.d/rhn-search + +%changelog +* Fri Jan 15 2010 Michael Mraka michael.mraka@redhat.com 0.8.1-1 +- rebuild for spacewalk 0.8 + +* Wed Dec 2 2009 Miroslav SuchÃœ msuchy@redhat.com 0.7.3-1 +- 480342 - move init script from /etc/init.d to /etc/rc.d/init.d + +* Wed Nov 25 2009 Miroslav SuchÃœ msuchy@redhat.com 0.7.1-1 +- 516872 - Fixed search server's log properties (paji@redhat.com) +- Update doc indexes to reside in "en-US" (jmatthew@redhat.com) +- bumping versions to 0.7.0 (jmatthew@redhat.com) + +* Thu Jun 25 2009 John Matthews jmatthew@redhat.com 0.6.11-1 +- 487014 - SystemSearch remove score requirement to redirect to SDC on 1 result + (jmatthew@redhat.com) +- DocumentationSearch fixing returned links so they are relative and update for + segment dir (jmatthew@redhat.com) + +* Mon Jun 01 2009 jesus m. rodriguez jesusr@redhat.com 0.6.10-1 +- added java debugger options for local dev config (jesusr@redhat.com) +- store pid file locally. (jesusr@redhat.com) +- 487014 - SystemSearch reducing scrore threshold for a single result to + redirect to SDC page (jmatthew@redhat.com) + +* Tue May 26 2009 Devan Goodwin dgoodwin@redhat.com 0.6.9-1 +- 501925 - "rhn-search cleanindex" will now restart search after cleaning index + (jmatthew@redhat.com) + +* Thu May 21 2009 jesus m. rodriguez jesusr@redhat.com 0.6.8-1 +- 457350 - added package search apis to match functionality webui provides (jmatthew@redhat.com) +- Fixing "free form" search. Adding a boolean flag which when passed (jmatthew@redhat.com) + +* Fri May 08 2009 jesus m. rodriguez jesusr@redhat.com 0.6.7-1 +- clean up javadoc in AdminHandler (jesusr@redhat.com) + +* Tue May 05 2009 jesus m. rodriguez jesusr@redhat.com 0.6.6-1 +- cleanup ScheduleManager, fix potential null pointer, added a testcase. + +* Wed Apr 22 2009 jesus m. rodriguez jesusr@redhat.com 0.6.5-1 +- Add /etc/rhn/search/ %dir directive in search spec. (dgoodwin@redhat.com) + +* Fri Apr 17 2009 Devan Goodwin dgoodwin@redhat.com 0.6.4-1 +- Search server adding a xmlrpc call that will allow updates to be triggered + immediately (jmatthew@redhat.com) +- 487209 - enhanced system search for hardware devices (jmatthew@redhat.com) +- 495921 - Limit results returned from Errata Search by Advisory + (jmatthew@redhat.com) +- 487158 - SystemSearch fixed search by customInfo (jmatthew@redhat.com) +- Search Server adding a debug option to print out explanation of searches + (jmatthew@redhat.com) +- 442439 - enhancing csv for systemsearch (jmatthew@redhat.com) +- 487189 - System Search fixed search by checkin (jmatthew@redhat.com) + +* Sun Apr 05 2009 jesus m. rodriguez jesusr@redhat.com 0.6.3-1 +- 487424 - add logrotate to %%files section (jesusr@redhat.com) + +* Sat Apr 04 2009 jesus m. rodriguez jesusr@redhat.com 0.6.2-1 +- install logrotate.d directory +- remove / after RPM_BUILD_ROOT it's not needed +- search requires doc-indexes, sw-doc-indexes provides doc-indexes (jesusr@redhat.com) +- 492624 - "rhn-search cleanindex" checks db connection is up (jmatthew@redhat.com) +- bump Versions to 0.6.0 (jesusr@redhat.com) + +* Tue Mar 31 2009 jesus m. rodriguez jesusr@redhat.com 0.5.10-1 +- 487424 - defined logrotate for rhn_search.log + +* Thu Mar 31 2009 jesus m. rodriguez jesusr@redhat.com +- 487424 - define logrotate for rhn_search.log + +* Thu Feb 26 2009 jesus m. rodriguez jesusr@redhat.com 0.5.9-1 +- systems query changed to no longer require anything more than rhnServer. + +* Thu Feb 19 2009 jesus m. rodriguez jesusr@redhat.com 0.5.8-1 +- Fixing problem which broke unique documents in the lucene index. +- SearchServer - SystemSearch lowering minimum score threshold + +* Thu Feb 19 2009 jesus m. rodriguez jesusr@redhat.com 0.5.7-1 +- 486182 - SystemSearch "id" is now tokenized, allows flexible 'ngram' s +- 485820 - System Search: Isn't reindexing when system data is modified + +* Thu Feb 12 2009 jesus m. rodriguez jesusr@redhat.com 0.5.6-1 +- 484610 - doc search results were reversed, displaying worst results first +- Cleaned up "tabs" which got added into some files previously. +- Removed "orig" as a field from DocResults + +* Thu Feb 05 2009 jesus m. rodriguez jesusr@redhat.com 0.5.5-1 +- 479541, 483867 - replaced runuser with /sbin/runuser + +* Mon Jan 26 2009 jesus m. rodriguez jesusr@redhat.com 0.5.4-1 +- allow search-server to run in degraded mode if nutch isn't installed +- fixing rpmlint warning: summary-ended-with-dot +- allow multiple language search + +* Fri Jan 23 2009 Devan Goodwin dgoodwin@redhat.com 0.5.3-1 +- Fix bad install command. + +* Thu Jan 22 2009 Dennis Gilmore dennis@ausil.us 0.5.2-1 +- update java-devel BuildRequires to 1.6.0 +- preserve timestamps on install + +* Wed Jan 21 2009 Milan Zazrivec mzazrivec@redhat.com 0.5.1-1 +- obsolete rhn-search before 5.3.0 (for satellite's benefit) + +* Tue Jan 13 2009 John Matthews jmatthews@redhat.com 0.4.10-1 +- Added SystemSearch parameter, running kernel + +* Thu Dec 18 2008 John Matthews jmatthews@redhat.com 0.4.7-1 +- rebuild for spacewalk 0.4 +- added doc search + +* Mon Dec 8 2008 John Matthews jmatthews@redhat.com 0.4.1-1 +- updates for "make test-srpm" to function + +* Fri Oct 24 2008 Jesus M. Rodriguez jesusr@redhat.com 0.3.4-1 +- rebuild +- added a "cleanindex" option to init.d script for rhn-search + +* Tue Oct 21 2008 Michael Mraka michael.mraka@redhat.com 0.3.3-1 +- resolves #467717 - fixed sysvinit scripts +- resolves #467877 - use runuser instead of su + +* Tue Sep 23 2008 Milan Zazrivec 0.3.1-1 +- fixed package obsoletes + +* Wed Sep 3 2008 Milan Zazrivec 0.2.6-1 +- config file needs to point to correct spacewalk-search.jar + +* Tue Sep 2 2008 Jesus Rodriguez 0.2.5-1 +- tagged for rebuild +- includes errata search capability +- fix setup and source0 to be name-version +- removed unnecessary bloat from libsrc directory +- removed apache-ibatis-sqlmap as a requires for now. FIXME + +* Mon Aug 11 2008 Jesus Rodriguez 0.1.2-1 +- tagged for rebuild after rename, also bumping version + +* Tue Aug 5 2008 Jan Pazdziora 0.1.2-0 +- tagged for rebuild after rename, also bumping version + +* Mon Aug 4 2008 Jan Pazdziora 0.1.1-0 +- rebuilt with BuildRequires: java-devel >= 1.5.0 diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/config.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/config.xml new file mode 100644 index 0000000..a41e673 --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/config.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMapConfig + PUBLIC "-//ibatis.apache.org//DTD SQL Map Config 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-config-2.dtd"> +<!-- Always ensure to use the correct XML header as above! --> +<sqlMapConfig> + <!-- These settings control SqlMap configuration details, primarily to do with transaction + management. They are all optional (see the Developer Guide for more). --> + <settings + cacheModelsEnabled="true" + statementCachingEnabled="false" + enhancementEnabled="true" + lazyLoadingEnabled="true" + maxRequests="100" + maxSessions="10" + maxTransactions="10" + useStatementNamespaces="false" + defaultStatementTimeout="30" + /> + <!-- Configure a datasource to use with this SQL Map using SimpleDataSource. + Notice the use of the properties from the above resource --> + <transactionManager type="JDBC" > + <dataSource type="SIMPLE"> + <property name="JDBC.Driver" value="${search.connection.driver_class}"/> + <property name="JDBC.ConnectionURL" value="${search.connection.url}"/> + <property name="JDBC.Username" value="${search.connection.username}"/> + <property name="JDBC.Password" value="${search.connection.password}"/> + </dataSource> + </transactionManager> + <!-- Identify all SQL Map XML files to be loaded by this SQL map. Notice the paths + are relative to the classpath. For now, we only have one
--> + <sqlMap resource="com/redhat/satellite/search/db/package.xml" /> + <sqlMap resource="com/redhat/satellite/search/db/errata.xml" /> + <sqlMap resource="com/redhat/satellite/search/db/server.xml" /> + <sqlMap resource="com/redhat/satellite/search/db/errata_handler.xml" /> + <sqlMap resource="com/redhat/satellite/search/db/hwdevice.xml" /> + <sqlMap resource="com/redhat/satellite/search/db/snapshot_tag.xml" /> + <sqlMap resource="com/redhat/satellite/search/db/server_custom_info.xml" /> +</sqlMapConfig> diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata.xml new file mode 100644 index 0000000..13c5985 --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata.xml @@ -0,0 +1,67 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMap + PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-2.dtd"> +<sqlMap namespace="errata"> + <select id="listErrataFromId" + parameterClass="long" + resultClass="com.redhat.satellite.search.db.models.Errata"> + SELECT + e.id, + e.advisory, + e.advisory_type as advisoryType, + e.advisory_name as advisoryName, + e.advisory_rel as advisoryRel, + e.product, + e.description, + e.synopsis, + e.topic, + e.solution, + e.issue_date as issueDate, + e.update_date as updateDate, + e.notes, + e.org_id, + e.created, + e.modified, + e.last_modified as lastModified + FROM rhnErrata e + WHERE e.id > #id# + ORDER by e.id ASC + </select> + <delete id="deleteLastErrata"> + DELETE rhnIndexerWork where object_type = 'errata' + </delete> + <insert id="createLastErrata" + parameterClass="java.util.Map"> + INSERT INTO rhnIndexerWork values ('errata', #id#, #last_modified#) + </insert> + <update id="updateLastErrata" + parameterClass="java.util.Map"> + UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'errata' + </update> + <select id="getLastErrataId" + resultClass="java.lang.Long"> + SELECT last_id from rhnIndexerWork where object_type='errata' + </select> + <select id="maxErrataId" resultClass="java.lang.Long"> + SELECT MAX(ID) + FROM rhnErrata + </select> + <select id="verifyErrataVisibility" resultClass="string"> + select distinct e.id + from rhnErrata e, + rhnChannelErrata ce, + rhnAvailableChannels ac, + pxrsessions ps, + web_contact wc + where e.id = ce.errata_id and + ce.channel_id = ac.channel_id and + ac.org_id = wc.org_id and + ps.web_user_id = wc.id and + ps.id = #sessionId# + and e.id IN + <iterate property="id_list" open="(" close=")" conjunction=","> + #id_list[]# + </iterate> + </select> +</sqlMap> diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata_handler.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata_handler.xml new file mode 100644 index 0000000..7c6cece --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/errata_handler.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMap + PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-2.dtd"> +<sqlMap namespace="errataHandler"> + + <select id="listErrataByCVE" + parameterClass="Map" + resultClass="com.redhat.satellite.search.index.Result"> + SELECT DISTINCT + ecve.errata_id as id + from rhnErrataCVE ecve, + rhnCVE cve, + rhnErrata e, + rhnChannelErrata ce, + rhnAvailableChannels ac, + pxtsessions ps, + web_contact wc + where cve.id = ecve.cve_id and + e.id = ce.errata_id and + e.id = ecve.errata_id and + ce.channel_id = ac.channel_id and + ac.org_id = wc.org_id and + ps.web_user_id = wc.id and + ps.id = #sessionId# and + cve.name = #param0# + </select> + + <select id="listErrataByIssueDateRange" + parameterClass="Map" + resultClass="com.redhat.satellite.search.index.Result" > + SELECT + E.id as id + FROM rhnErrata E, + rhnChannelErrata ce, + rhnAvailableChannels ac, + pxtsessions ps, + web_contact wc + WHERE + e.id = ce.errata_id and + ce.channel_id = ac.channel_id and + ac.org_id = wc.org_id and + ps.web_user_id = wc.id and + ps.id = #sessionId# and + E.issue_date >= TO_DATE(#param0#, 'YYYY-MM-DD HH24:MI:SS') + AND E.issue_date <= TO_DATE(#param1#, 'YYYY-MM-DD HH24:MI:SS') + ORDER BY E.issue_date ASC + </select> +</sqlMap> diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/hwdevice.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/hwdevice.xml new file mode 100644 index 0000000..70d6d8a --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/hwdevice.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMap + PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-2.dtd"> + +<sqlMap namespace="hwdevice"> + + <select id="getHardwareDeviceById" parameterClass="java.util.Map" + resultClass="com.redhat.satellite.search.db.models.HardwareDevice"> + SELECT + rd.id as id, + rd.server_id as serverId, + rd.class as classInfo, + rd.bus as bus, + rd.detached as detached, + rd.device as device, + rd.driver as driver, + rd.description as description, + rd.prop1 as vendorId, + rd.prop2 as deviceId, + rd.prop3 as subVendorId, + rd.prop4 as subDeviceId, + rd.pcitype as pciType + FROM rhnDevice rd + WHERE rd.id > #id# + ORDER by rd.id ASC + </select> + + + <delete id="deleteLastHardwareDevice"> + DELETE rhnIndexerWork where object_type = 'hwdevice' + </delete> + <insert id="createLastHardwareDevice" + parameterClass="java.util.Map"> + INSERT INTO rhnIndexerWork values ('hwdevice', #id#, #last_modified#) + </insert> + <update id="updateLastHardwareDevice" + parameterClass="java.util.Map"> + UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'hwdevice' + </update> + <select id="getLastHardwareDeviceId" + resultClass="java.lang.Long"> + SELECT last_id from rhnIndexerWork where object_type='hwdevice' + </select> + <select id="getLastHardwareDeviceIndexRun" + resultClass="java.util.Date"> + SELECT last_modified from rhnIndexerWork where object_type='hwdevice' + </select> + <select id="queryAllHwDeviceIds" resultClass="Long"> + SELECT id from rhnDevice + </select> + +</sqlMap> diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/package.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/package.xml new file mode 100644 index 0000000..46e84d5 --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/package.xml @@ -0,0 +1,85 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMap + PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-2.dtd"> +<sqlMap namespace="package"> + <select id="getPackageById" + parameterClass="long" + resultClass="com.redhat.satellite.search.db.models.RhnPackage"> + SELECT + p.id as id, + pn.name as name, + pe.epoch as epoch, + pe.version as version, + pe.release as release, + pa.label as arch, + p.description as description, + p.summary as summary + FROM rhnPackage p, + rhnPackageName pn, + rhnPackageEVR pe, + rhnPackageArch pa + WHERE p.evr_id = pe.id + and p.name_id = pn.id + and p.package_arch_id = pa.id + and p.id = #id# + </select> + <select id="listPackagesFromId" + parameterClass="long" + resultClass="com.redhat.satellite.search.db.models.RhnPackage"> + SELECT + p.id as id, + pn.name as name, + pe.epoch as epoch, + pe.version as version, + pe.release as release, + pa.label as arch, + p.description as description, + p.summary as summary + FROM rhnPackage p, + rhnPackageName pn, + rhnPackageEVR pe, + rhnPackageArch pa + WHERE p.evr_id = pe.id + and p.name_id = pn.id + and p.package_arch_id = pa.id + and p.id > #id# + ORDER by p.id ASC + </select> + <delete id="deleteLastPackage"> + DELETE rhnIndexerWork where object_type = 'package' + </delete> + <insert id="createLastPackage" + parameterClass="java.util.Map"> + INSERT INTO rhnIndexerWork values ('package', #id#, #last_modified#) + </insert> + <update id="updateLastPackage" + parameterClass="java.util.Map"> + UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'package' + </update> + <select id="getLastPackageId" + resultClass="java.lang.Long"> + SELECT last_id from rhnIndexerWork where object_type='package' + </select> + <select id="maxPackageId" resultClass="java.lang.Long"> + SELECT MAX(ID) + FROM rhnPackage + </select> + <select id="verifyPackageVisibility" resultClass="string"> + select distinct p.id + from rhnPackage p, + rhnAvailableChannels ac, + rhnChannelPackage cp, + pxtsessions ps, + web_contact wc + where cp.package_id = p.id + and ac.channel_id = cp.channel_id + and ac.org_id = wc.org_id + and ps.web_user_id = wc.id + and ps.id = #session_id# + and p.id IN + <iterate property="id_list" open="(" close=")" conjunction=","> + #id_list[]# + </iterate> + </select> +</sqlMap> diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server.xml new file mode 100644 index 0000000..03f8b13 --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server.xml @@ -0,0 +1,105 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMap + PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-2.dtd"> +<sqlMap namespace="server"> + <select id="getServerByIdOrDate" parameterClass="java.util.Map" + resultClass="com.redhat.satellite.search.db.models.Server"> + SELECT + s.id as id, + s.name as name, + s.description as description, + s.info as info, + s.created as registered, + s.running_kernel as runningKernel, + sl.machine as machine, + sl.rack as rack, + sl.room as room, + sl.building as building, + sl.address1 as address1, + sl.address2 as address2, + sl.city as city, + sl.state as state, + sl.country as country, + rsn.hostname as hostname, + rsn.ipaddr as ipaddr, + dmi.vendor as dmiVendor, + dmi.system as dmiSystem, + dmi.product as dmiProduct, + dmi.bios_vendor as dmiBiosVendor, + dmi.bios_version as dmiBiosVersion, + dmi.bios_release as dmiBiosRelease, + dmi.asset as dmiAsset, + dmi.board as dmiBoard, + cpu.bogomips as cpuBogoMIPS, + cpu.cache as cpuCache, + cpu.family as cpuFamily, + cpu.mhz as cpuMhz, + cpu.stepping as cpuStepping, + cpu.flags as cpuFlags, + cpu.model as cpuModel, + cpu.version as cpuVersion, + cpu.vendor as cpuVendor, + cpu.nrcpu as cpuNumberOfCpus, + cpu.acpiversion as cpuAcpiVersion, + cpu.apic as cpuApic, + cpu.apmversion as cpuApmVersion, + cpu.chipset as cpuChipset, + si.checkin as checkin, + rr.ram as ram, + rr.swap as swap + FROM rhnServer s + LEFT OUTER JOIN rhnServerNetwork rsn ON s.id = rsn.server_id + LEFT OUTER JOIN rhnServerInfo si on s.id = si.server_id + LEFT OUTER JOIN rhnRam rr on s.id = rr.server_id + LEFT OUTER JOIN rhnServerLocation sl ON s.id = sl.server_id + LEFT OUTER JOIN rhnServerDmi dmi ON s.id = dmi.server_id + LEFT OUTER JOIN rhnCPU cpu on s.id = cpu.server_id + WHERE s.modified >= #last_modified# + or rsn.modified >= #last_modified# + or si.checkin >= #last_modified# + or rr.modified >= #last_modified# + or sl.modified >= #last_modified# + or dmi.modified >= #last_modified# + or cpu.modified >= #last_modified# + or s.id > #id# + ORDER by s.id ASC + </select> + + <delete id="deleteLastServer"> + DELETE rhnIndexerWork where object_type = 'server' + </delete> + <insert id="createLastServer" + parameterClass="java.util.Map"> + INSERT INTO rhnIndexerWork values ('server', #id#, #last_modified#) + </insert> + <update id="updateLastServer" + parameterClass="java.util.Map"> + UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'server' + </update> + <select id="getLastServerId" + resultClass="java.lang.Long"> + SELECT last_id from rhnIndexerWork where object_type='server' + </select> + <select id="getLastServerIndexRun" + resultClass="java.util.Date"> + SELECT last_modified from rhnIndexerWork where object_type='server' + </select> + <select id="verifyServerVisibility" resultClass="string"> + SELECT object_id + FROM rhnVisibleObjects + WHERE pxt_session_id = #session_id# + AND object_type = #object_type# + AND object_id IN + <iterate property="id_list" open="(" close=")" conjunction=","> + #id_list[]# + </iterate> + </select> + <select id="maxServerId" resultClass="java.lang.Long"> + SELECT MAX(ID) + FROM rhnServer + </select> + <select id="queryAllServerIds" resultClass="Long"> + SELECT id from rhnServer + </select> +</sqlMap> diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server_custom_info.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server_custom_info.xml new file mode 100644 index 0000000..c56090c --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/server_custom_info.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMap + PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-2.dtd"> + +<sqlMap namespace="serverCustomInfo"> + + <select id="getServerCustomInfoByIdOrDate" parameterClass="java.util.Map" + resultClass="com.redhat.satellite.search.db.models.ServerCustomInfo"> + + SELECT + rscdv.key_id as keyId, + rscdv.server_id as serverId, + rscdv.value as value, + rscdv.created_by as createdBy, + rscdv.last_modified_by as lastModifiedBy, + rcdk.label as label, + TO_CHAR(rscdv.created, 'YYYY-MM-DD HH24-MI-SS') as created, + TO_CHAR(rscdv.modified, 'YYYY-MM-DD HH24-MI-SS') as modified + FROM rhnServerCustomDataValue rscdv + INNER JOIN rhnCustomDataKey rcdk on rcdk.id = rscdv.key_id + WHERE rscdv.modified > #last_modified# + ORDER by rscdv.key_id ASC + </select> + + <delete id="deleteLastServerCustomInfo"> + DELETE rhnIndexerWork where object_type = 'server_custom_info' + </delete> + <insert id="createLastServerCustomInfo" + parameterClass="java.util.Map"> + INSERT INTO rhnIndexerWork values ('server_custom_info', #id#, #last_modified#) + </insert> + <update id="updateLastServerCustomInfo" + parameterClass="java.util.Map"> + UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'server_custom_info' + </update> + <select id="getLastServerCustomInfoId" + resultClass="java.lang.Long"> + SELECT last_id from rhnIndexerWork where object_type='server_custom_info' + </select> + <select id="getLastServerCustomInfoIndexRun" + resultClass="java.util.Date"> + SELECT last_modified from rhnIndexerWork where object_type='server_custom_info' + </select> + <select id="queryAllServerCustomInfoIds" resultClass="com.redhat.satellite.search.db.models.ServerCustomInfo"> + SELECT server_id as serverId, key_id as keyId from rhnServerCustomDataValue + </select> +</sqlMap> diff --git a/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/snapshot_tag.xml b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/snapshot_tag.xml new file mode 100644 index 0000000..5d4d67c --- /dev/null +++ b/search-server/spacewalk-search/src/config/com/redhat/satellite/search/db/snapshot_tag.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE sqlMap + PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" + "http://ibatis.apache.org/dtd/sql-map-2.dtd"> + +<sqlMap namespace="snapshotTag"> + + <select id="getSnapshotTagByIdOrDate" parameterClass="java.util.Map" + resultClass="com.redhat.satellite.search.db.models.SnapshotTag"> + + SELECT + rt.id as id, + rst.snapshot_id as snapshotId, + rtn.id as tagNameId, + rst.server_id as serverId, + rt.org_id as orgId, + rtn.name as name, + TO_CHAR(rst.created, 'YYYY-MM-DD HH24-MI-SS') as created, + TO_CHAR(rst.modified, 'YYYY-MM-DD HH24-MI-SS') as modified + FROM rhnSnapshotTag rst + INNER JOIN rhnTag rt on rst.tag_id = rt.id + INNER JOIN rhnTagName rtn on rt.name_id = rtn.id + WHERE rst.tag_id > #id# OR rst.modified > #last_modified# + ORDER by rst.snapshot_id ASC + </select> + + + <delete id="deleteLastSnapshotTag"> + DELETE rhnIndexerWork where object_type = 'snapshot_tag' + </delete> + <insert id="createLastSnapshotTag" + parameterClass="java.util.Map"> + INSERT INTO rhnIndexerWork values ('snapshot_tag', #id#, #last_modified#) + </insert> + <update id="updateLastSnapshotTag" + parameterClass="java.util.Map"> + UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'snapshot_tag' + </update> + <select id="getLastSnapshotTagId" + resultClass="java.lang.Long"> + SELECT last_id from rhnIndexerWork where object_type='snapshot_tag' + </select> + <select id="getLastSnapshotTagIndexRun" + resultClass="java.util.Date"> + SELECT last_modified from rhnIndexerWork where object_type='snapshot_tag' + </select> + + <select id="queryAllSnapshotTagIds" resultClass="Long"> + SELECT snapshot_id from rhnSnapshotTag + </select> + + +</sqlMap> diff --git a/search-server/spacewalk-search/src/config/etc/logrotate.d/rhn-search b/search-server/spacewalk-search/src/config/etc/logrotate.d/rhn-search new file mode 100644 index 0000000..a2793cb --- /dev/null +++ b/search-server/spacewalk-search/src/config/etc/logrotate.d/rhn-search @@ -0,0 +1,9 @@ +/var/log/rhn/search/rhn_search.log { + weekly + rotate 5 + copytruncate + compress + notifempty + missingok + size=10M +} diff --git a/search-server/spacewalk-search/src/config/log4j.properties b/search-server/spacewalk-search/src/config/log4j.properties new file mode 100644 index 0000000..6c2195a --- /dev/null +++ b/search-server/spacewalk-search/src/config/log4j.properties @@ -0,0 +1,35 @@ +log4j.appender.RootAppender=org.apache.log4j.ConsoleAppender +log4j.appender.RootAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.RootAppender.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.rootLogger=WARN,RootAppender + +## XmlRpc Logging +log4j.appender.SearchAppender=org.apache.log4j.FileAppender +log4j.appender.SearchAppender.file=/var/log/rhn/search/rhn_search.log +log4j.appender.SearchAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.SearchAppender.layout.ConversionPattern=[%d] %-5p - %m%n +log4j.logger.com.redhat.satellite=WARN,SearchAppender +log4j.logger.org.quartz=WARN,SearchAppender +#log4j.logger.net.rkbloom.logdriver.LogPreparedStatement=DEBUG +#log4j.logger.net.rkbloom.logdriver.LogStatement=DEBUG +#log4j.logger.net.rkbloom.logdriver.LogCallableStatement=DEBUG +#log4j.logger.net.rkbloom.logdriver.LogConnection=DEBUG +#log4j.logger.com.redhat.satellite.search.rpc.handlers=DEBUG,SearchAppender +#log4j.logger.com.redhat.satellite.search.index.IndexManager=DEBUG,SearchAppender +#log4j.logger.com.redhat.satellite.search.ngram.NGramQueryParser=DEBUG,SearchAppender +#log4j.logger.com.redhat.satellite.search.scheduler.tasks=DEBUG,SearchAppender +#log4j.logger.com.redhat.satellite.search.rpc.handlers=DEBUG,SearchAppender + +# SqlMap logging configuration... +#log4j.logger.com.ibatis=DEBUG +#log4j.logger.com.ibatis.common.jdbc.SimpleDataSource=DEBUG +#log4j.logger.com.ibatis.sqlmap.engine.cache.CacheModel=DEBUG +#log4j.logger.com.ibatis.sqlmap.engine.impl.SqlMapClientImpl=DEBUG +#log4j.logger.com.ibatis.sqlmap.engine.builder.xml.SqlMapParser=DEBUG +#log4j.logger.com.ibatis.common.util.StopWatch=DEBUG +#log4j.logger.java.sql.Connection=DEBUG +#log4j.logger.java.sql.Statement=DEBUG +#log4j.logger.java.sql.PreparedStatement=DEBUG +#log4j.logger.java.sql.ResultSet=DEBUG + + diff --git a/search-server/spacewalk-search/src/config/log4j.test.properties b/search-server/spacewalk-search/src/config/log4j.test.properties new file mode 100644 index 0000000..72e7ac6 --- /dev/null +++ b/search-server/spacewalk-search/src/config/log4j.test.properties @@ -0,0 +1,17 @@ +log4j.appender.RootAppender=org.apache.log4j.ConsoleAppender +log4j.appender.RootAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.RootAppender.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.appender.RootAppender.Threshold=WARN +log4j.rootLogger=INFO,RootAppender + +log4j.appender.SearchAppender=org.apache.log4j.FileAppender +log4j.appender.SearchAppender.file=test_rhn_search.log +log4j.appender.SearchAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.SearchAppender.layout.ConversionPattern=[%d] %-5p - %m%n +log4j.appender.SearchAppender.Threshold=DEBUG + +log4j.logger.com.redhat.satellite=INFO,SearchAppender +log4j.logger.com.redhat.satellite.search.config=INFO + +log4j.logger.java.sql.ResultSet=WARN +log4j.logger.com.redhat.satellite.search.index.ngram.tests=DEBUG,RootAppender diff --git a/search-server/spacewalk-search/src/config/rhn-search b/search-server/spacewalk-search/src/config/rhn-search new file mode 100644 index 0000000..f7856b8 --- /dev/null +++ b/search-server/spacewalk-search/src/config/rhn-search @@ -0,0 +1,395 @@ +#! /bin/sh +# +# rhn-search This shell script takes care of starting and stopping +# the java rhn-search server. +# +# chkconfig: - 94 12 +# description: This is Spacewalk search engine. +# +### BEGIN INIT INFO +# Provides: rhn-search +# Required-Start: $local_fs $network satellite-httpd +# Required-Stop: $local_fs $network satellite-httpd +# Default-Start: +# Default-Stop: 0 1 2 3 4 5 6 +# Short-Description: This is Spacewalk search engine. +# Description: This is Spacewalk search engine. +### END INIT INFO + +# +# Skeleton sh script suitable for starting and stopping +# wrapped Java apps on the Solaris platform. +# +# Make sure that PIDFILE points to the correct location, +# if you have changed the default location set in the +# wrapper configuration file. +# + +#----------------------------------------------------------------------------- +# These settings can be modified to fit the needs of your application + +# Application +APP_NAME="rhn-search" +APP_LONG_NAME="rhn-search" + +# Wrapper +WRAPPER_CMD="/usr/bin/rhnsearchd" +WRAPPER_CONF="/etc/rhn/search/rhn_search_daemon.conf" + +# Priority at which to run the wrapper. See "man nice" for valid priorities. +# nice is only used if a priority is specified. +PRIORITY= + +# Location of the pid file. +PIDDIR="/var/run" + +# If uncommented, causes the Wrapper to be shutdown using an anchor file. +# When launched with the 'start' command, it will also ignore all INT and +# TERM signals. +#IGNORE_SIGNALS=true + +# If specified, the Wrapper will be run as the specified user when the 'start' +# command is passed to this script. When running with the 'console' command +# the current user will be used. +# IMPORTANT - Make sure that the user has the required privileges to write +# the PID file and wrapper.log files. Failure to be able to write the log +# file will cause the Wrapper to exit without any way to write out an error +# message. +# NOTE - This will set the user which is used to run the Wrapper as well as +# the JVM and is not useful in situations where a privileged resource or +# port needs to be allocated prior to the user being changed. +#RUN_AS_USER= + +unlock() { + IDXDIR=`grep search.index_work_dir /etc/rhn/rhn.conf` + if [ -z $IDXDIR ] + then + IDXDIR=`grep search.index_work_dir /etc/rhn/search/rhn_search.conf` + fi + + if [ ! -z $IDXDIR ] + then + IDXDIR=`echo $IDXDIR | sed -e 's/=/ /g' | awk '{print $2}'` + find $IDXDIR -type f -name write.lock -exec rm -f {} ; + fi +} + +removeindex() { + SEARCH_CLASS_PATH=`build-classpath-directory /usr/share/rhn/search/lib /usr/share/java/ojdbc14.jar` + CLASSPATH=${SEARCH_CLASS_PATH} java com.redhat.satellite.search.DeleteIndexes +} + +# Do not modify anything beyond this point +#----------------------------------------------------------------------------- + +# Get the fully qualified path to the script +case $0 in + /*) + SCRIPT="$0" + ;; + *) + PWD=`pwd` + SCRIPT="$PWD/$0" + ;; +esac + +# Change spaces to ":" so the tokens can be parsed. +SCRIPT=`echo $SCRIPT | sed -e 's; ;:;g'` +# Get the real path to this script, resolving any symbolic links +TOKENS=`echo $SCRIPT | sed -e 's;/; ;g'` +REALPATH= +for C in $TOKENS; do + REALPATH="$REALPATH/$C" + while [ -h "$REALPATH" ] ; do + LS="`ls -ld "$REALPATH"`" + LINK="`expr "$LS" : '.*-> (.*)$'`" + if expr "$LINK" : '/.*' > /dev/null; then + REALPATH="$LINK" + else + REALPATH="`dirname "$REALPATH"`""/$LINK" + fi + done +done +# Change ":" chars back to spaces. +REALPATH=`echo $REALPATH | sed -e 's;:; ;g'` + +# Change the current directory to the location of the script +cd "`dirname "$REALPATH"`" + +# Process ID +ANCHORFILE="$PIDDIR/$APP_NAME.anchor" +PIDFILE="$PIDDIR/$APP_NAME.pid" +pid="" + +# Resolve the location of the 'ps' command +PSEXE="/usr/bin/ps" +if [ ! -x $PSEXE ] +then + PSEXE="/bin/ps" + if [ ! -x $PSEXE ] + then + echo "Unable to locate 'ps'." + echo "Please report this message along with the location of the command on your system." + exit 1 + fi +fi + +# Build the nice clause +if [ "X$PRIORITY" = "X" ] +then + CMDNICE="" +else + CMDNICE="nice -$PRIORITY" +fi + +# Check the configured user +if [ "X$RUN_AS_USER" != "X" ] +then + # Resolve the location of the 'id' command + IDEXE="/usr/xpg4/bin/id" + if [ ! -x $IDEXE ] + then + IDEXE="/usr/bin/id" + if [ ! -x $IDEXE ] + then + echo "Unable to locate 'id'." + echo "Please report this message along with the location of the command on your system." + exit 1 + fi + fi + + if [ "`$IDEXE -u -n`" = "$RUN_AS_USER" ] + then + # Already running as the configured user. Avoid password prompts by not calling runuser. + RUN_AS_USER="" + fi +fi + +getpid() { + if [ -f $PIDFILE ] + then + if [ -r $PIDFILE ] + then + pid=`cat $PIDFILE` + if [ "X$pid" != "X" ] + then + # Verify that a process with this pid is still running. + pid=`$PSEXE -p $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` + if [ "X$pid" = "X" ] + then + # This is a stale pid file. + rm -f $PIDFILE + echo "Removed stale pid file: $PIDFILE" + fi + fi + else + echo "Cannot read $PIDFILE." + exit 1 + fi + fi +} + +testpid() { + pid=`$PSEXE -p $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` + if [ "X$pid" = "X" ] + then + # Process is gone so remove the pid file. + rm -f $PIDFILE + fi +} + +console() { + echo "Running $APP_LONG_NAME..." + getpid + if [ "X$pid" = "X" ] + then + if [ "X$IGNORE_SIGNALS" = "X" ] + then + exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE + else + exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.anchorfile=$ANCHORFILE + fi + else + echo "$APP_LONG_NAME is already running." + exit 1 + fi +} + +start() { + echo "Starting $APP_LONG_NAME..." + getpid + if [ "X$pid" = "X" ] + then + if [ "X$IGNORE_SIGNALS" = "X" ] + then + if [ "X$RUN_AS_USER" = "X" ] + then + exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.daemonize=TRUE + else + /sbin/runuser -m $RUN_AS_USER -c "exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.daemonize=TRUE" + fi + else + if [ "X$RUN_AS_USER" = "X" ] + then + exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.anchorfile=$ANCHORFILE wrapper.ignore_signals=TRUE wrapper.daemonize=TRUE + else + /sbin/runuser -m $RUN_AS_USER -c "exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.anchorfile=$ANCHORFILE wrapper.ignore_signals=TRUE wrapper.daemonize=TRUE" + fi + fi + else + echo "$APP_LONG_NAME is already running." + exit 1 + fi +} + +stopit() { + echo "Stopping $APP_LONG_NAME..." + getpid + if [ "X$pid" = "X" ] + then + echo "$APP_LONG_NAME was not running." + else + if [ "X$IGNORE_SIGNALS" = "X" ] + then + # Running so try to stop it. + kill $pid + if [ $? -ne 0 ] + then + # An explanation for the failure should have been given + echo "Unable to stop $APP_LONG_NAME." + exit 1 + fi + else + rm -f $ANCHORFILE + if [ -f $ANCHORFILE ] + then + # An explanation for the failure should have been given + echo "Unable to stop $APP_LONG_NAME." + exit 1 + fi + fi + + # We can not predict how long it will take for the wrapper to + # actually stop as it depends on settings in wrapper.conf. + # Loop until it does. + savepid=$pid + CNT=0 + TOTCNT=0 + while [ "X$pid" != "X" ] + do + # Loop for up to 5 minutes + if [ "$TOTCNT" -lt "300" ] + then + if [ "$CNT" -lt "5" ] + then + CNT=`expr $CNT + 1` + else + echo "Waiting for $APP_LONG_NAME to exit..." + CNT=0 + fi + TOTCNT=`expr $TOTCNT + 1` + + sleep 1 + + testpid + else + pid= + fi + done + + pid=$savepid + testpid + if [ "X$pid" != "X" ] + then + echo "Timed out waiting for $APP_LONG_NAME to exit." + echo " Attempting a forced exit..." + kill -9 $pid + fi + + pid=$savepid + testpid + if [ "X$pid" != "X" ] + then + echo "Failed to stop $APP_LONG_NAME." + exit 1 + else + echo "Stopped $APP_LONG_NAME." + fi + fi +} + +status() { + getpid + if [ "X$pid" = "X" ] + then + echo "$APP_LONG_NAME is not running." + exit 1 + else + echo "$APP_LONG_NAME is running ($pid)." + exit 0 + fi +} + +dump() { + echo "Dumping $APP_LONG_NAME..." + getpid + if [ "X$pid" = "X" ] + then + echo "$APP_LONG_NAME was not running." + + else + kill -3 $pid + + if [ $? -ne 0 ] + then + echo "Failed to dump $APP_LONG_NAME." + exit 1 + else + echo "Dumped $APP_LONG_NAME." + fi + fi +} + +case "$1" in + + 'console') + console + ;; + + 'start') + start + ;; + + 'stop') + stopit + unlock + ;; + + 'restart') + stopit + unlock + start + ;; + + 'status') + status + ;; + + 'dump') + dump + ;; + + 'cleanindex') + stopit + unlock + removeindex + start + ;; + + *) + echo "Usage: $0 { console | start | stop | restart | status | dump | cleanindex }" + exit 1 + ;; +esac + +exit 0 diff --git a/search-server/spacewalk-search/src/config/search/rhn_search.conf b/search-server/spacewalk-search/src/config/search/rhn_search.conf new file mode 100644 index 0000000..6a3cb7d --- /dev/null +++ b/search-server/spacewalk-search/src/config/search/rhn_search.conf @@ -0,0 +1,15 @@ +search.index_work_dir=/usr/share/rhn/search/indexes +search.rpc_handlers=index:com.redhat.satellite.search.rpc.handlers.IndexHandler,db:com.redhat.satellite.search.rpc.handlers.DatabaseHandler,admin:com.redhat.satellite.search.rpc.handlers.AdminHandler +search.max_hits_returned=500 +search.connection.driver_class=oracle.jdbc.driver.OracleDriver +search.score_threshold=.10 +search.system_score_threshold=.01 +search.errata_score_threshold=.20 +search.errata.advisory_score_threshold=.30 +search.min_ngram = 1 +search.max_ngram = 5 +search.doc.limit_results = false +search.schedule.interval = 300000 +search.log.explain.results = false + + diff --git a/search-server/spacewalk-search/src/config/search/rhn_search_daemon.conf b/search-server/spacewalk-search/src/config/search/rhn_search_daemon.conf new file mode 100644 index 0000000..07d7145 --- /dev/null +++ b/search-server/spacewalk-search/src/config/search/rhn_search_daemon.conf @@ -0,0 +1,86 @@ +#******************************************************************** +# Wrapper Properties +#******************************************************************** +# Java Application +wrapper.java.command=/usr/bin/java + +# Java Main class. This class must implement the WrapperListener interface +# or guarantee that the WrapperManager class is initialized. Helper +# classes are provided to do this for you. See the Integration section +# of the documentation for details. +wrapper.java.mainclass=com.redhat.satellite.search.Main + +# Java Classpath (include wrapper.jar) Add class path elements as +# needed starting from 1 +wrapper.java.classpath.1=/usr/share/rhn/search/classes/ +wrapper.java.classpath.2=/usr/share/rhn/search/lib/commons-lang.jar +wrapper.java.classpath.3=/usr/share/rhn/search/lib/commons-logging.jar +wrapper.java.classpath.4=/usr/share/rhn/search/lib/ibatis-2.3.0.677.jar +wrapper.java.classpath.5=/usr/share/rhn/search/lib/log4j.jar +wrapper.java.classpath.6=/usr/share/rhn/search/lib/lucene-core-2.3.2.jar +wrapper.java.classpath.7=/usr/share/java/ojdbc14.jar +wrapper.java.classpath.8=/usr/share/rhn/search/lib/picocontainer-1.3.jar +wrapper.java.classpath.9=/usr/share/rhn/search/lib/quartz.jar +wrapper.java.classpath.10=/usr/share/rhn/search/lib/redstone-xmlrpc.jar +wrapper.java.classpath.11=/usr/share/rhn/search/lib/redstone-xmlrpc-client.jar +wrapper.java.classpath.12=/usr/share/rhn/search/lib/simple-core-3.1.3.jar +wrapper.java.classpath.13=/usr/share/rhn/search/lib/spacewalk-search.jar +wrapper.java.classpath.14=/usr/share/java/tanukiwrapper.jar +wrapper.java.classpath.15=/usr/share/rhn/search/lib/lucene-analyzers-2.3.2.jar +wrapper.java.classpath.16=/usr/share/rhn/search/lib/hadoop-0.18.1-core.jar +# +#Remember we want nutch/conf to be found ahead of the nutch.jar +#this ensures the config files used are ours and not the default ones +#in the jar. +# +wrapper.java.classpath.17=/usr/share/nutch +wrapper.java.classpath.18=/usr/share/nutch/conf +wrapper.java.classpath.19=/usr/share/rhn/search/lib/nutch-2008-12-01_04-01-21.jar +wrapper.java.classpath.20=/usr/share/rhn/search/lib/commons-httpclient-3.0.jar + +# Java Library Path (location of Wrapper.DLL or libwrapper.so) +wrapper.java.library.path.1=/usr/lib +wrapper.java.library.path.2=/usr/lib64 + +# Java Additional Parameters +#wrapper.java.additional.1= + +# Initial Java Heap Size (in MB) +#wrapper.java.initmemory=32 + +# Maximum Java Heap Size (in MB) +#wrapper.java.maxmemory=96 + +# Application parameters. Add parameters as needed starting from 1 +#wrapper.app.parameter.1=<YourMainClass> + +#******************************************************************** +# Wrapper Logging Properties +#******************************************************************** +# Format of output for the console. (See docs for formats) +wrapper.console.format=PM + +# Log Level for console output. (See docs for log levels) +wrapper.console.loglevel=INFO + +# Log file to use for wrapper output logging. +wrapper.logfile=/var/log/rhn/search/rhn_search_daemon.log + +# Format of output for the log file. (See docs for formats) +wrapper.logfile.format=LPTM + +# Log Level for log file output. (See docs for log levels) +wrapper.logfile.loglevel=INFO + +# Maximum size that the log file will be allowed to grow to before +# the log is rolled. Size is specified in bytes. The default value +# of 0, disables log rolling. May abbreviate with the 'k' (kb) or +# 'm' (mb) suffix. For example: 10m = 10 megabytes. +wrapper.logfile.maxsize=5m + +# Maximum number of rolled log files which will be allowed before old +# files are deleted. The default value of 0 implies no limit. +wrapper.logfile.maxfiles=0 + +# Log Level for sys/event log output. (See docs for log levels) +wrapper.syslog.loglevel=NONE diff --git a/search-server/spacewalk-search/src/config/test.properties b/search-server/spacewalk-search/src/config/test.properties new file mode 100644 index 0000000..a37fd98 --- /dev/null +++ b/search-server/spacewalk-search/src/config/test.properties @@ -0,0 +1,5 @@ +# test config file DO NOT SHIP +search.connection.username = spacewalk +search.connection.password = spacewalk +search.connection.url=jdbc:log:oracle.jdbc.driver.OracleDriver:oracle:thin:@localhost:1521:xe +search.connection.driver_class=net.rkbloom.logdriver.LogDriver diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/ContainerRunner.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/ContainerRunner.java new file mode 100644 index 0000000..65e089a --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/ContainerRunner.java @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search; + +import org.picocontainer.defaults.DefaultPicoContainer; + +/** + * Runs a PicoContainer instance in a separate thread + * to prevent the main process from exiting + * + * @version $Rev$ + */ +class ContainerRunner implements Runnable { + + private DefaultPicoContainer container; + + /** + * Constructor + * @param containerIn PicoContainer instance to use + */ + ContainerRunner(DefaultPicoContainer containerIn) { + container = containerIn; + } + + /** + * {@inheritDoc} + */ + public void run() { + container.start(); + try { + synchronized (this) { + this.wait(); + } + container.stop(); + } + catch (InterruptedException e) { + container.stop(); + } + } + + /** + * Stop the thread + */ + public void stop() { + synchronized (this) { + this.notify(); + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/DeleteIndexes.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/DeleteIndexes.java new file mode 100644 index 0000000..81206ab --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/DeleteIndexes.java @@ -0,0 +1,187 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.WriteQuery; +import com.redhat.satellite.search.config.Configuration; + + +/** + * Reindex - cleans up indexes on filesystem and database so reindexing will occur + * @version $Rev: 1 $ + */ +public class DeleteIndexes { + private static Logger log = Logger.getLogger(DeleteIndexes.class); + + private DeleteIndexes() { + } + + protected static boolean deleteDirectory(File dir) { + File[] files = dir.listFiles(); + boolean warning = true; + for (int i = 0; i < files.length; i++) { + if (files[i].isDirectory()) { + deleteDirectory(files[i]); + } + if (files[i].delete()) { + log.debug("Deleted: " + files[i].getAbsolutePath()); + } + else { + log.warn("*ERROR* unable to delete: " + files[i].getAbsolutePath()); + warning = false; + } + } + if (!dir.delete()) { + log.warn("*ERROR* unable to delete: " + dir.getAbsolutePath()); + warning = false; + } + return warning; + } + + protected static boolean deleteIndexPath(String path) { + File dir = new File(path); + if ("/".equals(dir.getAbsolutePath())) { + log.warn("Error, passed in path is <" + path + "> this looks wrong"); + return false; + } + if (!dir.exists()) { + log.debug("Path <" + dir.getAbsolutePath() + "> doesn't exist"); + return true; // dir doesn't exist, so just as good as deleted + } + if (!dir.isDirectory()) { + log.warn("Error, passed in path <" + path + "> is not a directory"); + return false; + } + log.info("Attempting to delete " + dir.getAbsolutePath()); + return deleteDirectory(dir); + } + + protected static void deleteQuery(DatabaseManager databaseManager, + String queryName) throws SQLException { + WriteQuery query = null; + log.info("Running query: " + queryName); + query = databaseManager.getWriterQuery(queryName); + query.delete(null); + query.close(); + } + + /** + * @param args + */ + public static void main(String[] args) { + try { + Configuration config = new Configuration(); + DatabaseManager databaseManager = new DatabaseManager(config); + String indexWorkDir = config.getString("search.index_work_dir", null); + if (StringUtils.isBlank(indexWorkDir)) { + log.warn("Couldn't find path for where index files are stored."); + log.warn("Looked in config for property: search.index_work_dir"); + return; + } + List<IndexInfo> indexes = new ArrayList<IndexInfo>(); + indexes.add(new IndexInfo("deleteLastErrata", + indexWorkDir + File.separator + "errata")); + indexes.add(new IndexInfo("deleteLastPackage", + indexWorkDir + File.separator + "package")); + indexes.add(new IndexInfo("deleteLastServer", + indexWorkDir + File.separator + "server")); + indexes.add(new IndexInfo("deleteLastHardwareDevice", + indexWorkDir + File.separator + "hwdevice")); + indexes.add(new IndexInfo("deleteLastSnapshotTag", + indexWorkDir + File.separator + "snapshotTag")); + indexes.add(new IndexInfo("deleteLastServerCustomInfo", + indexWorkDir + File.separator + "serverCustomInfo")); + for (IndexInfo info : indexes) { + deleteQuery(databaseManager, info.getQueryName()); + if (!deleteIndexPath(info.getDirPath())) { + log.warn("Failed to delete index for " + info.getDirPath()); + } + } + } + catch (SQLException e) { + log.error("Caught Exception: ", e); + if (e.getErrorCode() == 17002) { + log.error("Unable to establish database connection."); + log.error("Ensure database is available and connection details are " + + "correct, then retry"); + } + System.exit(1); + } + catch (IOException e) { + log.error("Caught Exception: ", e); + System.exit(1); + } + log.info("Index files have been deleted and database has been cleaned up, " + + "ready to reindex"); + } + + /** + * IndexInfo + */ + protected static class IndexInfo { + protected String queryName; + protected String dirPath; + + /** + * + * @param query query name to delete all records + * @param path string pointing to where index files reside + */ + public IndexInfo(String query, String path) { + queryName = query; + dirPath = path; + } + + /** + * Set the query name to delete all records for this index + * @param queryNameIn name of the query + */ + public void setQueryName(String queryNameIn) { + queryName = queryNameIn; + } + /** + * Returns the query name + * @return + */ + public String getQueryName() { + return queryName; + } + /** + * Set the path where the index files exist + * @param path String which points to index files + */ + public void setDirPath(String path) { + dirPath = path; + } + /** + * Returns the path + * @return a string which points to the index files + */ + public String getDirPath() { + return dirPath; + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/Main.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/Main.java new file mode 100644 index 0000000..9ff0d9d --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/Main.java @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search; + +import com.redhat.satellite.search.config.Configuration; +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.rpc.RpcServer; +import com.redhat.satellite.search.scheduler.ScheduleManager; + +import org.apache.log4j.Logger; +import org.picocontainer.defaults.DefaultPicoContainer; +import org.tanukisoftware.wrapper.WrapperListener; +import org.tanukisoftware.wrapper.WrapperManager; + +/** + * Entry point for the Tanuki daemon wrapper + * + * @version $Rev$ + */ +public class Main implements WrapperListener { + + private static Logger log = Logger.getLogger(Main.class); + private static final Class[] COMPONENTS = {DatabaseManager.class, + IndexManager.class, + RpcServer.class, + ScheduleManager.class}; + + private DefaultPicoContainer container; + private ContainerRunner runner; + + /** + * {@inheritDoc} + */ + public void controlEvent(int arg) { + } + + /** + * {@inheritDoc} + */ + public Integer start(String[] argv) { + Configuration config = new Configuration(); + container = new DefaultPicoContainer(); + container.registerComponentInstance(config); + for (int x = 0; x < COMPONENTS.length; x++) { + container.registerComponentImplementation(COMPONENTS[x]); + } + runner = new ContainerRunner(container); + Thread t = new Thread(runner); + t.setDaemon(false); + t.start(); + + return null; + } + + /** + * {@inheritDoc} + */ + public int stop(int arg) { + log.info("Stopping Main"); + runner.stop(); + return 0; + } + + /** + * Main entry point + * @param argv command-line args + */ + public static void main(String[] argv) { + Main m = new Main(); + WrapperManager.start(m, argv); + if (log.isDebugEnabled()) { + log.debug("Returned from WrapperManager.start"); + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/ConfigException.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/ConfigException.java new file mode 100644 index 0000000..c83c7ea --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/ConfigException.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +/* + * AUTOMATICALLY GENERATED FILE, DO NOT EDIT. + */ +package com.redhat.satellite.search.config; + +/** + * Base class for all Config exceptions + * <p> + + * + * @version definition($Rev: 76724 $)/template($Rev: 67725 $) + */ +public class ConfigException extends RuntimeException { + + /** + * Constructor + * @param message exception message + */ + public ConfigException(String message) { + super(message); + } + + /** + * Constructor + * @param message exception message + * @param cause the cause (which is saved for later retrieval + * by the Throwable.getCause() method). (A null value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public ConfigException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/Configuration.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/Configuration.java new file mode 100644 index 0000000..1971b26 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/Configuration.java @@ -0,0 +1,622 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.config; + +import com.redhat.satellite.search.config.translator.TranslatorRegistry; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.TreeSet; + +/** + * The Config class acts as an abstraction layer between our configuration + * system, and the actual implementation. The basic idea is that there is a + * global config, but you can instantiate one of your own if you want. This + * layer insulates us from changing the underlying implementation. + * <p> + * Config files are properties, with /etc/rhn/default/rhn.conf setting defaults + * that can be overridden /etc/rhn/rhn.conf. + * + * @version $Rev$ + */ +public class Configuration { + + private static Logger logger = Logger.getLogger(Configuration.class); + + // + // Location of config files + // + + /** + * The default directory in which to look for config files + */ + public static final String DEFAULT_CONF_DIR = "/etc/rhn"; + + /** + * The system property containing the RHN configuration directory. + * If the property is not set, config files are read + * from {@link #DEFAULT_CONF_DIR} + */ + private static final String RHN_CONF_DIR_PROPERTY = "rhn.config.dir"; + + /** + * The system property containing the SearchServer configuration directory. + * If the property is not set, config files are read + * from {@link #DEFAULT_CONF_DIR}/search + */ + private static final String SEARCH_CONF_DIR_PROPERTY = "search.config.dir"; + + + /** + * List of values that are considered true, ignoring case. + */ + private static final String[] TRUE_VALUES = {"1", "y", "true", "yes", "on"}; + + /** + * array of prefix in the order they should be search + * if the given lookup string is without a namespace. + */ + private String[] prefixOrder = new String[] {"web", "server"}; + + /** hash of configuration properties */ + private Properties configValues = new Properties(); + + private List<KeyTranslator> translators; + + /** set of configuration file names */ + private TreeSet<File> fileList = new TreeSet<File>(new Comparator() { + + /** {inheritDoc} */ + public int compare(Object o1, Object o2) { + // Need to make sure we read the child namespace before the base + // namespace. To do that, we sort the list in reverse order based + // on the length of the file name. If two filenames have the same + // length, then we need to do a lexigraphical comparison to make + // sure that the filenames themselves are different. + + File f1 = (File) o1; + File f2 = (File) o2; + int lenDif = f2.getAbsolutePath().length() - f1.getAbsolutePath().length(); + + if (lenDif != 0) { + return lenDif; + } + return f2.compareTo(f1); + } + }); + + /** + * public constructor. Rereads config entries every time it is called. + * + * @throws ConfigException error from the Configuration layers. the jakarta + * commons conf system just throws Exception, which makes it hard to react. + * sometimes it is an IOExceptions, sometimes a SAXParserException, + * sometimes a VindictiveException. so we just turn them into our own + * exception type and toss them up. as we discover ones we might + * meaningfully want to react to, we can specialize ConfigException and catch + * those + */ + public Configuration() throws ConfigException { + translators = TranslatorRegistry.getTranslators(); + addPath(getSearchConfigDir()); + addPath(getDefaultConfigFilePath()); + parseFiles(); + } + + /** + * Construct from a Reader. + * @param rdr Configuration data in Reader format. + * @throws ConfigException thrown if a problem reading the configuration + * occurs. + */ + public Configuration(Reader rdr) throws ConfigException { + translators = TranslatorRegistry.getTranslators(); + ReaderWrapper rdrwrap = new ReaderWrapper(rdr); + Properties props = new Properties(); + try { + parseStream(props, "search", rdrwrap); + } + catch (IOException ioe) { + throw new ConfigException(ioe.getMessage(), ioe); + } + } + + /** + * Add a path to the config object for parsing + * @param path The path to add + */ + public void addPath(String path) { + addFiles(path); + } + + /** + * static method to get the singleton Config option + * + * @return the config option + */ +// public static synchronized Config get() { +// if (singletonConfig == null) { +// singletonConfig = new Config(); +// } +// return singletonConfig; +// } + + private static String getRHNConfigDir() { + String confDir = System.getProperty(RHN_CONF_DIR_PROPERTY); + + if (StringUtils.isBlank(confDir)) { + confDir = DEFAULT_CONF_DIR; + } + return confDir; + } + + private static String getSearchConfigDir() { + String confDir = System.getProperty(SEARCH_CONF_DIR_PROPERTY); + + if (StringUtils.isBlank(confDir)) { + confDir = getRHNConfigDir() + "/search"; + } + return confDir; + } + + /** + * Get the path to the rhn.conf file we use. + * + * @return String path. + */ + public static String getDefaultConfigFilePath() { + return getRHNConfigDir() + "/rhn.conf"; + } + + /** + * Get the configuration entry for the given string name. If the value + * is null, then return the given defValue. defValue can be null as well. + * @param name name of property + * @param defValue default value for property if it is null. + * @return the value of the property with the given name, or defValue. + */ + public String getString(String name, String defValue) { + String ret = getString(name); + if (ret == null) { + if (logger.isDebugEnabled()) { + logger.debug("getString() - returning default value"); + } + ret = defValue; + } + return ret; + } + + /** + * Returns a map of the values with the given name. The value must be + * in the format of key:value otherwise, an empty map is returned. + * @param name name of property + * @return map of the values of the property matching k:v. + */ + public Map<String, String> getMap(String name) { + String value = getString(name); + Map retval = null; + if (value != null) { + retval = new HashMap<String, String>(); + String[] pairs = value.split(","); + for (int x = 0; x < pairs.length; x++) { + String[] nv = pairs[x].split(":"); + if (nv.length != 2) { + continue; + } + retval.put(nv[0].trim(), nv[1].trim()); + } + } + return retval; + } + + /** + * get the config entry for string s + * + * @param value string to get the value of + * @return the value + */ + public String getString(String value) { + if (logger.isDebugEnabled()) { + logger.debug("getString() - getString() called with: " + value); + } + if (value == null) { + return null; + } + + int lastDot = value.lastIndexOf("."); + String ns = ""; + String property = value; + if (lastDot > 0) { + property = value.substring(lastDot + 1); + ns = value.substring(0, lastDot); + } + if (logger.isDebugEnabled()) { + logger.debug("getString() - getString() -> Getting property: " + + property); + } + String result = configValues.getProperty(property); + if (logger.isDebugEnabled()) { + logger.debug("getString() - getString() -> result: " + result); + } + if (result == null) { + if (!"".equals(ns)) { + result = configValues.getProperty(ns + "." + property); + } + else { + for (int i = 0; i < prefixOrder.length; i++) { + result = configValues.getProperty(prefixOrder[i] + "." + property); + if (result != null) { + break; + } + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("getString() - getString() -> returning: " + result); + } + + if (result == null || result.equals("")) { + return null; + } + + return stripComments(result); + } + + private String stripComments(String string) { + // check for null + if (string == null) { + return null; + } + + int breakpos = string.indexOf('#'); + + // check for comment in value + if (breakpos < 0) { + return string.trim(); + } + + while (breakpos > -1) { + if (breakpos == 0) { + string = ""; + break; + } + else if (string.charAt(breakpos - 1) == '\') { + breakpos = string.indexOf('#', breakpos + 1); + } + else { + string = string.substring(0, breakpos); + break; + } + } + + if (string.length() == 0) { + return string; + } + + return string.replaceAll("\\#", "#").trim(); + } + + /** + * get the config entry for string s + * + * @param s string to get the value of + * @return the value + */ + public int getInt(String s) { + return getInt(s, 0); + } + + /** + * get the config entry for string s, if no value is found + * return the defaultValue specified. + * + * @param s string to get the value of + * @param defaultValue Default value if entry is not found. + * @return the value + */ + public int getInt(String s, int defaultValue) { + Integer val = getInteger(s); + if (val == null) { + return defaultValue; + } + return val.intValue(); + } + + /** + * get the config entry for string s + * + * @param s string to get the value of + * @return the value + */ + public double getDouble(String s) { + return getDouble(s, 0.0); + } + + /** + * get the config entry for string s, if no value is found + * return the defaultValue specified. + * + * @param s string to get the value of + * @param defaultValue Default value if entry is not found. + * @return the value + */ + public double getDouble(String s, double defaultValue) { + String val = getString(s); + if (val == null) { + return defaultValue; + } + return new Double(val).doubleValue(); + } + + /** + * get the config entry for string s + * + * @param s string to get the value of + * @return the value + */ + public Integer getInteger(String s) { + String val = getString(s); + if (val == null) { + return null; + } + return new Integer(val); + } + + /** + * Parses a comma-delimited list of values as a java.util.List + * @param name config entry name + * @return instance of java.util.List populated with config values + */ + public List getList(String name) { + List retval = new LinkedList(); + String[] vals = getStringArray(name); + if (vals != null) { + retval.addAll(Arrays.asList(vals)); + } + return retval; + } + + /** + * get the config entry for string s + * + * @param s string to get the value of + * @return the value + */ + public String[] getStringArray(String s) { + if (s == null) { + return null; + } + String value = getString(s); + + if (value == null) { + return null; + } + + return value.split(","); + } + + /** + * get the config entry for string name + * + * @param name string to set the value of + * @param value new value + * @return the previous value of the property + */ + public String setString(String name, String value) { + return (String) configValues.setProperty(name, value); + } + + /** + * get the config entry for string s + * + * @param s string to get the value of + * @return the value + */ + public boolean getBoolean(String s) { + String value = getString(s); + if (logger.isDebugEnabled()) { + logger.debug("getBoolean() - " + s + " is : " + value); + } + if (value == null) { + return false; + } + + //need to check the possible true values + // tried to use BooleanUtils, but that didn't + // get the job done for an integer as a String. + + + for (int i = 0; i < TRUE_VALUES.length; i++) { + if (TRUE_VALUES[i].equalsIgnoreCase(value)) { + if (logger.isDebugEnabled()) { + logger.debug("getBoolean() - Returning true: " + value); + } + return true; + } + } + + return false; + } + + /** + * set the config entry for string name + * @param s string to set the value of + * @param b new value + */ + public void setBoolean(String s, String b) { + // need to check the possible true values + // tried to use BooleanUtils, but that didn't + // get the job done for an integer as a String. + for (int i = 0; i < TRUE_VALUES.length; i++) { + if (TRUE_VALUES[i].equalsIgnoreCase(b)) { + configValues.setProperty(s, "1"); + + // get out we're done here + return; + } + } + configValues.setProperty(s, "0"); + } + + private void addFiles(String path) { + File f = new File(path); + + if (f.isDirectory()) { + // bugzilla: 154517; only add items that end in .conf + File[] files = f.listFiles(); + for (int i = 0; i < files.length; i++) { + if (files[i].getName().endsWith((".conf"))) { + fileList.add(files[i]); + } + } + } + else { + fileList.add(f); + } + } + + private String makeNamespace(File f) { + String ns = f.getName(); + + // This is really hokey, but it works. Basically, rhn.conf doesn't + // match the standard rhn_foo.conf convention. So, to create the + // namespace, we first special case rhn.* + if (ns.startsWith("rhn.")) { + return ""; + } + + ns = ns.replaceFirst("rhn_", ""); + ns = ns.substring(0, ns.lastIndexOf('.')); + ns = ns.replaceAll("_", "."); + return ns; + } + + private void parseStream(Properties props, String namespace, InputStream is) + throws IOException { + + props.load(is); + + // loop through all of the config values in the properties file + // making sure the prefix is there. + Properties newProps = new Properties(); + for (Iterator j = props.keySet().iterator(); j.hasNext();) { + String key = (String) j.next(); + String newKey = key; + if (!key.startsWith(namespace)) { + newKey = namespace + "." + key; + } + logger.debug("Adding: " + newKey + ": " + props.getProperty(key)); + // translate the original key + newKey = translateKey(key); + newProps.put(newKey, props.getProperty(key)); + } + configValues.putAll(newProps); + } + + /** + * Parse all of the added files. + */ + public void parseFiles() { + for (Iterator i = fileList.iterator(); i.hasNext();) { + File curr = (File) i.next(); + parseFile(curr); + } + } + + private void parseFile(File file) { + Properties props = new Properties(); + String ns = makeNamespace(file); + logger.debug("Adding namespace: " + ns + " for file: " + + file.getAbsolutePath()); + + try { + parseStream(props, ns, new FileInputStream(file)); + } + catch (IOException e) { + logger.error("Could not parse file" + file, e); + } + } + + /** + * Returns a subset of the properties for the given namespace. This is + * not a particularly fast method and should be used only at startup or + * some other discreet time. Repeated calls to this method are guaranteed + * to be slow. + * @param namespace Namespace of properties to be returned. + * @return subset of the properties that begin with the given namespace. + */ + public Properties getNamespaceProperties(String namespace) { + Properties prop = new Properties(); + for (Iterator i = configValues.keySet().iterator(); i.hasNext();) { + String key = (String) i.next(); + if (key.startsWith(namespace)) { + if (logger.isDebugEnabled()) { + logger.debug("Looking for key: [" + key + "]"); + } + prop.put(key, configValues.getProperty(key)); + } + } + return prop; + } + + private String translateKey(String key) { + for (Iterator itr = translators.iterator(); itr.hasNext();) { + KeyTranslator trans = (KeyTranslator) itr.next(); + if (trans.shouldTranslate(key)) { + key = trans.translateKey(key); + } + } + + return key; + } + + private static class ReaderWrapper extends InputStream { + private Reader rdr; + + public ReaderWrapper(Reader reader) { + rdr = reader; + } + + /** + * {@inheritDoc} + */ + @Override + public int read() throws IOException { + return rdr.read(); + } + + @Override + public void close() throws IOException { + super.close(); + rdr.close(); + } + } +} + diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/KeyTranslator.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/KeyTranslator.java new file mode 100644 index 0000000..26f442a --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/KeyTranslator.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.config; + + +/** + * Translator + * @version $Rev$ + */ +public interface KeyTranslator { + + /** + * Translates the given key into something else. That something else is + * determined by the specific implementation. The value for the given key + * should not be affected. + * @param key key to be translated. + * @return The translated key. + */ + String translateKey(String key); + + /** + * Returns true if the key should be translated. + * @param key to be translated + * @return true if the key should be translated. + */ + boolean shouldTranslate(String key); +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java new file mode 100644 index 0000000..9ac2283 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java @@ -0,0 +1,137 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.config.tests; + +import com.redhat.satellite.search.config.Configuration; + +import java.io.BufferedReader; +import java.io.StringReader; +import java.util.List; +import java.util.Map; + +import junit.framework.TestCase; + + +/** + * ConfigurationTest + * @version $Rev$ + */ +public class ConfigurationTest extends TestCase { + + private Configuration config; + + @Override + protected void setUp() throws Exception { + StringBuilder builder = new StringBuilder(); + builder.append("search.boolean=true\n"); + builder.append("search.index_work_dir=/tmp/search-server-test\n"); + builder.append("search.rpc_port=2828\n"); + builder.append("search.list=item1,item2, item3\n"); + builder.append("hibernate.connection.username=rhnsat\n"); + builder.append("search.rpc_handlers="); + builder.append( + "index:com.redhat.satellite.search.rpc.handlers.IndexHandler\n\n"); + StringReader sr = new StringReader(builder.toString()); + config = new Configuration(new BufferedReader(sr)); + } + + public void testGetValue() { + assertEquals("bar", config.getString("foo", "bar")); + assertEquals( + "index:com.redhat.satellite.search.rpc.handlers.IndexHandler", + config.getString("search.rpc_handlers", "")); + assertNull(config.getString(null, null)); + assertEquals("", config.getString(null, "")); + } + + public void testGetValueAsInt() { + assertEquals(2828, config.getInt("search.rpc_port")); + assertEquals(2828, config.getInt("search.rpc_port", 0)); + assertEquals(10, config.getInt("search.nothere", 10)); + assertEquals(10, config.getInt(null, 10)); + try { + config.getInt("search.rpc_handlers", 0); + fail("getValueAsInt didn't throw exception"); + } + catch (Exception e) { + // expected + } + } + + public void testGetValuesAsMap() { + assertNull(config.getMap("search.notthere")); + assertNull(config.getMap(null)); + + Map<String, String> map = config.getMap("search.rpc_handlers"); + assertNotNull(map); + assertEquals("com.redhat.satellite.search.rpc.handlers.IndexHandler", + map.get("index")); + + map = config.getMap("search.rpc_port"); + assertNotNull(map); + assertTrue(map.keySet().isEmpty()); + } + + public void testGetValues() { + assertNotNull(config.getList("search.notthere")); + assertTrue(config.getList("search.notthere").isEmpty()); + assertNotNull(config.getList(null)); + assertTrue(config.getList(null).isEmpty()); + + List<String> list = config.getList("search.list"); + assertNotNull(list); + assertEquals(3, list.size()); + assertEquals("item1", list.get(0)); + assertEquals("item2", list.get(1)); + assertEquals(" item3", list.get(2)); + } + + public void testTranslation() { + assertNull(config.getString("hibernate.connection.username")); + assertEquals("rhnsat", config.getString("search.connection.username")); + } + + public void testGetBoolean() { + assertTrue(config.getBoolean("search.boolean")); + config.setBoolean("search.boolean", "false"); + assertFalse(config.getBoolean("search.boolean")); + } + + public void testGetDefaultConfig() { + String confDir = System.getProperty("rhn.config.dir"); + if (confDir == null || "".equals(confDir)) { + confDir = "/etc/rhn"; + } + assertEquals(confDir + "/rhn.conf", + Configuration.getDefaultConfigFilePath()); + } + + public void testSetString() { + config.setString("search.stringvalue", "foobar"); + assertEquals("foobar", config.getString("search.stringvalue")); + } + + public void testGetDouble() { + config.setString("search.double", ".30"); + assertEquals(0.0, config.getDouble("better.not.be.a.double")); + assertEquals(.30, config.getDouble("search.double")); + + config.setString("search.double", "30"); + assertEquals(30.0, config.getDouble("search.double")); + + config.setString("search.double", ""); + assertEquals(0.0, config.getDouble("search.double")); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java new file mode 100644 index 0000000..b3eb094 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.config.translator; + +import com.redhat.satellite.search.config.KeyTranslator; + + +/** + * HibernateToSearchTranslator + * @version $Rev$ + */ +public class HibernateToSearchTranslator implements KeyTranslator { + + /** + * {@inheritDoc} + */ + public String translateKey(String key) { + if (key.equals("hibernate.connection.username") || + key.equals("hibernate.connection.password") || + key.equals("hibernate.connection.driver_class") || + key.equals("hibernate.connection.url")) { + return key.replaceAll("hibernate", "search"); + } + + return key; + } + + /** + * {@inheritDoc} + */ + public boolean shouldTranslate(String key) { + return (key.equals("hibernate.connection.username") || + key.equals("hibernate.connection.password") || + key.equals("hibernate.connection.driver_class") || + key.equals("hibernate.connection.url")); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java new file mode 100644 index 0000000..4564812 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.config.translator; + +import com.redhat.satellite.search.config.KeyTranslator; + +import java.util.LinkedList; +import java.util.List; + + +/** + * TranslatorRegistry + * @version $Rev$ + */ +public class TranslatorRegistry { + + private TranslatorRegistry() { + // hide the default constructor + } + + private static final List<Class> TRANSLATOR_CLASSES; + static { + TRANSLATOR_CLASSES = new LinkedList<Class>(); + TRANSLATOR_CLASSES.add(HibernateToSearchTranslator.class); + } + + private static List<KeyTranslator> translators = new LinkedList<KeyTranslator>(); + + /** + * Returns the list of all available translators. + * @return the list of all available translators. + */ + public static List<KeyTranslator> getTranslators() { + if (translators.isEmpty() && !TRANSLATOR_CLASSES.isEmpty()) { + for (Class clazz : TRANSLATOR_CLASSES) { + try { + Object s = clazz.newInstance(); + translators.add((KeyTranslator)s); + } + catch (Exception e) { + e.printStackTrace(System.out); + } + } + } + + return translators; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Connection.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Connection.java new file mode 100644 index 0000000..5325a56 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Connection.java @@ -0,0 +1,97 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.db; + +import com.ibatis.sqlmap.client.SqlMapSession; + +import org.apache.log4j.Logger; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +/** + * Simplified connection class for executing arbitrary SQL + * + * @version $Rev$ + */ +public class Connection { + private static Logger log = Logger.getLogger(Connection.class); + + private SqlMapSession session; + + /** + * Constructor + * @param sess iBatis session to use + */ + public Connection(SqlMapSession sess) { + session = sess; + } + + /** + * Close down all resources + * @throws SQLException something bad happened + */ + public void close() throws SQLException { + session.close(); + } + + /** + * Executes a query + * @param statement string of valid SQL + * @param handler called on each row + * @throws SQLException something bad happened + */ + public void executeQuery(String statement, ResultHandler handler) throws SQLException { + java.sql.Connection cn = null; + Statement stmt = null; + ResultSet rs = null; + try { + log.warn("start transaction"); + session.startTransaction(); + log.warn("get current connection"); + cn = session.getCurrentConnection(); + log.warn("creating statement"); + stmt = cn.createStatement(); + log.warn("execute query"); + rs = stmt.executeQuery(statement); + log.warn("process resultset"); + while (rs.next()) { + handler.handleRow(rs); + } + } + finally { + try { + log.warn("commit transaction"); + session.commitTransaction(); + } + finally { + try { + if (rs != null) { + log.warn("close RS"); + rs.close(); + } + } + finally { + if (stmt != null) { + log.warn("close STMT"); + stmt.close(); + } + } + } + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/DatabaseManager.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/DatabaseManager.java new file mode 100644 index 0000000..93ad0a9 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/DatabaseManager.java @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.db; + +import com.redhat.satellite.search.config.Configuration; + +import com.ibatis.sqlmap.client.SqlMapClient; +import com.ibatis.sqlmap.client.SqlMapClientBuilder; +import com.ibatis.sqlmap.client.SqlMapSession; + +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Properties; + +/** + * Manages DB activity - connections, running queries, etc + * @version $Rev$ + */ +public class DatabaseManager { + + private SqlMapClient client = null; + + /** + * Constructor + * @param config + * @throws IOException + */ + public DatabaseManager(Configuration config) throws IOException { + String configPath = config.getString("search.db_config_path", + "classpath"); + Reader reader = null; + if (configPath.equals("classpath")) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + InputStream stream = cl + .getResourceAsStream("com/redhat/satellite/search/db/config.xml"); + if (stream == null) { + throw new IllegalArgumentException( + "com/redhat/satellite/search/db/" + + "config.xml resource missing"); + } + reader = new InputStreamReader(stream); + } + else { + reader = new FileReader(configPath); + } + Properties overrides = config.getNamespaceProperties("search"); + client = SqlMapClientBuilder.buildSqlMapClient(reader, overrides); + } + + /** + * Open a named select query + * @param <T> type returned by query + * @param name name of query + * @return query object + */ + public <T> Query<T> getQuery(String name) { + SqlMapSession session = client.openSession(); + return new Query<T>(session, name); + } + + /** + * Open a named write (insert, update, delete) query + * @param name of query + * @return query object + */ + public WriteQuery getWriterQuery(String name) { + SqlMapSession session = client.openSession(); + return new WriteQuery(session, name); + } + + /** + * Opens a direct DB connection + * @return connection object + */ + public Connection getConnection() { + return new Connection(client.openSession()); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Query.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Query.java new file mode 100644 index 0000000..57c209e --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/Query.java @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.db; + +import com.ibatis.sqlmap.client.SqlMapSession; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * Named Select query + * @param <T> type returned by query + * + * @version $Rev$ + */ +public class Query<T> { + + private SqlMapSession session; + private String queryName; + + Query(SqlMapSession sessionIn, String queryNameIn) { + session = sessionIn; + queryName = queryNameIn; + } + + /** + * Close query and associated resources + * @throws SQLException something bad happened + */ + public void close() throws SQLException { + try { + try { + session.commitTransaction(); + } + catch (SQLException e) { + session.endTransaction(); + } + } + finally { + session.close(); + } + } + + /** + * Load list of objects from query + * @param param query param + * @return list of T + * @throws SQLException something bad happened + */ + @SuppressWarnings("unchecked") + public List<T> loadList(Object param) throws SQLException { + List r = session.queryForList(queryName, param); + List<Object> results = (List<Object>)r; + List<T> retval = new ArrayList<T>(results.size()); + for (Object item : results) { + retval.add((T)item); + } + return retval; + } + + /** + * Load single object from query + * @param param query apram + * @return T + * @throws SQLException something bad happened + */ + @SuppressWarnings("unchecked") + public T load(Object param) throws SQLException { + return (T)session.queryForObject(queryName, param); + } + + /** + * Load single object from query + * @return T + * @throws SQLException something bad happened + */ + @SuppressWarnings("unchecked") + public T load() throws SQLException { + return (T)session.queryForObject(queryName); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/ResultHandler.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/ResultHandler.java new file mode 100644 index 0000000..caf86cb --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/ResultHandler.java @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + + +package com.redhat.satellite.search.db; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Callback interface for Connection-based queries + * + * @version $Rev$ + */ +public interface ResultHandler { + + /** + * Called for each row in a resultset + * @param rs resultset + * @throws SQLException something bad happened + */ + void handleRow(ResultSet rs) throws SQLException; +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/WriteQuery.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/WriteQuery.java new file mode 100644 index 0000000..c394316 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/WriteQuery.java @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.db; + +import com.ibatis.sqlmap.client.SqlMapSession; + +import java.sql.SQLException; + +/** + * Query for writing (insert, update, delete) to a database + * + * @version $Rev$ + */ +public class WriteQuery { + + private SqlMapSession session; + private String queryName; + + WriteQuery(SqlMapSession sessionIn, String queryNameIn) { + session = sessionIn; + queryName = queryNameIn; + } + + /** + * Close query and all associated resources + * @throws SQLException something bad happened + */ + public void close() throws SQLException { + try { + try { + session.commitTransaction(); + } + catch (SQLException e) { + session.endTransaction(); + } + } + finally { + session.close(); + } + } + + /** + * Execute update query + * @param param query param + * @return number of rows updated + * @throws SQLException something bad happened + */ + public int update(Object param) throws SQLException { + return session.update(queryName, param); + } + + /** + * Execute delete query + * @param param query param + * @return number of rows updated + * @throws SQLException something bad happened + */ + public int delete(Object param) throws SQLException { + return session.delete(queryName, param); + } + + /** + * Execute insert query + * @param param query param + * @throws SQLException something bad happened + */ + public void insert(Object param) throws SQLException { + session.insert(queryName, param); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Errata.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Errata.java new file mode 100644 index 0000000..2a6d170 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Errata.java @@ -0,0 +1,323 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.db.models; + + +/** + * Errata + * @version $Rev$ + */ +public class Errata { + private long id; + private String advisory; + private String advisoryType; + private String advisoryName; + private long advisoryRel; + private String product; + private String description; + private String synopsis; + private String topic; + private String solution; + private String issueDate; + private String updateDate; + private String notes; + private String orgId; + private String created; + private String modified; + private String lastModified; + + /** + * Returns a string representation of the errata + * @return string representation of the errata + */ + public String toString() { + StringBuffer text = new StringBuffer("Errata<" + id + ", " + product + ">: "); + text.append(advisoryType + ", " + advisoryName + ", " + advisory); + text.append(", " + topic + ", " + synopsis); + return text.toString(); + } + + /** + * Returns the erratum's id. + * @return the erratum's id. + */ + public long getId() { + return id; + } + + /** + * The erratum's id. + * @param idIn erratum id. + */ + public void setId(long idIn) { + id = idIn; + } + + /** + * Returns the advisory. + * @return the advisory. + */ + public String getAdvisory() { + return advisory; + } + + /** + * Sets the advisory. + * @param advisoryIn + */ + public void setAdvisory(String advisoryIn) { + advisory = advisoryIn; + } + + /** + * Returns advisory name + * @return advisory name + */ + public String getAdvisoryName() { + return advisoryName; + } + + /** + * Sets advisory name + * @param advisoryNameIn new advisory name + */ + public void setAdvisoryName(String advisoryNameIn) { + advisoryName = advisoryNameIn; + } + + /** + * Returns advisory release + * @return advisory release + */ + public long getAdvisoryRel() { + return advisoryRel; + } + + /** + * Sets advisory release + * @param advisoryRelIn advisory release + */ + public void setAdvisoryRel(long advisoryRelIn) { + advisoryRel = advisoryRelIn; + } + + /** + * Returns advisory type + * @return advisory type + */ + public String getAdvisoryType() { + return advisoryType; + } + + /** + * Set advisory type + * @param advisoryTypeIn new advisory type + */ + public void setAdvisoryType(String advisoryTypeIn) { + advisoryType = advisoryTypeIn; + } + + /** + * Returns created date as a string + * @return created date as a string + */ + public String getCreated() { + return created; + } + + /** + * Sets the created date (expecting string format) + * @param createdIn string formatted date. + */ + public void setCreated(String createdIn) { + created = createdIn; + } + + /** + * Returns the errata description + * @return the errata description + */ + public String getDescription() { + return description; + } + + /** + * Sets the description + * @param descriptionIn description of errata + */ + public void setDescription(String descriptionIn) { + description = descriptionIn; + } + + /** + * Returns the erratum's issue date as a string. + * @return the erratum's issue date as a string. + */ + public String getIssueDate() { + return issueDate; + } + + /** + * Sets the issue date (expecting string format). + * @param issueDateIn string formatted date. + */ + public void setIssueDate(String issueDateIn) { + issueDate = issueDateIn; + } + + /** + * Returns the last time the erratum was modified as a string. + * @return the last time the erratum was modified as a string. + */ + public String getLastModified() { + return lastModified; + } + + /** + * sets the last modified date as a string. + * @param lastModifiedIn string formatted last modified date. + */ + public void setLastModified(String lastModifiedIn) { + lastModified = lastModifiedIn; + } + + /** + * Returns the modified date as a string. + * @return the modified date as a string. + */ + public String getModified() { + return modified; + } + + /** + * Sets the modified date as a string. + * @param modifiedIn string formatted date. + */ + public void setModified(String modifiedIn) { + modified = modifiedIn; + } + + /** + * Returns notes associated with the errataum. + * @return notes associated with the errataum. + */ + public String getNotes() { + return notes; + } + + /** + * Sets the notes for the erratum. + * @param notesIn notes. + */ + public void setNotes(String notesIn) { + notes = notesIn; + } + + /** + * The org id owning the errata. + * @return org id owning the errata. + */ + public String getOrgId() { + return orgId; + } + + /** + * Sets the orgid owning the errata. + * @param orgIdIn owning orgid. + */ + public void setOrgId(String orgIdIn) { + orgId = orgIdIn; + } + + /** + * Products affected. + * @return affected products. + */ + public String getProduct() { + return product; + } + + /** + * sets the product. + * @param productIn product + */ + public void setProduct(String productIn) { + product = productIn; + } + + /** + * Returns what the errata proposed solution. + * @return what the errata proposed solution. + */ + public String getSolution() { + return solution; + } + + /** + * Proposed solution + * @param solutionIn proposed solution. + */ + public void setSolution(String solutionIn) { + solution = solutionIn; + } + + /** + * Returns erratum's synopsis. + * @return erratum's synopsis. + */ + public String getSynopsis() { + return synopsis; + } + + /** + * Sets the erratum's synopsis. + * @param synopsisIn synopsis. + */ + public void setSynopsis(String synopsisIn) { + synopsis = synopsisIn; + } + + /** + * returns erratum topic. + * @return erratum topic. + */ + public String getTopic() { + return topic; + } + + /** + * Sets the erratum's topic. + * @param topicIn topic + */ + public void setTopic(String topicIn) { + topic = topicIn; + } + + /** + * Returns the update date as a string. + * @return the update date as a string. + */ + public String getUpdateDate() { + return updateDate; + } + + /** + * Sets the update date as a string. + * @param updateDateIn string formatted date. + */ + public void setUpdateDate(String updateDateIn) { + updateDate = updateDateIn; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/GenericRecord.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/GenericRecord.java new file mode 100644 index 0000000..fc51f60 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/GenericRecord.java @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.db.models; + +/** + * Server + * @version $Rev$ + */ +public abstract class GenericRecord { + protected long id = 0; + /** + * @return the id + */ + public long getId() { + return id; + } + /** + * @param idIn the id to set + */ + public void setId(long idIn) { + this.id = idIn; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java new file mode 100644 index 0000000..e4e82b7 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java @@ -0,0 +1,183 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.db.models; + +/** + * Server + * @version $Rev$ + */ +public class HardwareDevice extends GenericRecord { + + private long serverId; + private String classInfo; + private String bus; + private long detached; + private String device; + private String driver; + private String description; + private String vendorId; + private String deviceId; + private String subVendorId; + private String subDeviceId; + private long pciType; + + /** + * @return the serverId + */ + public long getServerId() { + return serverId; + } + /** + * @param serverIdIn the serverId to set + */ + public void setServerId(long serverIdIn) { + this.serverId = serverIdIn; + } + /** + * @return the classInfo + */ + public String getClassInfo() { + return classInfo; + } + /** + * @param classInfoIn the classInfo to set + */ + public void setClassInfo(String classInfoIn) { + this.classInfo = classInfoIn; + } + /** + * @return the bus + */ + public String getBus() { + return bus; + } + /** + * @param busIn the bus to set + */ + public void setBus(String busIn) { + this.bus = busIn; + } + /** + * @return the detached + */ + public long getDetached() { + return detached; + } + /** + * @param detachedIn the detached to set + */ + public void setDetached(long detachedIn) { + this.detached = detachedIn; + } + /** + * @return the device + */ + public String getDevice() { + return device; + } + /** + * @param deviceIn the device to set + */ + public void setDevice(String deviceIn) { + this.device = deviceIn; + } + /** + * @return the driver + */ + public String getDriver() { + return driver; + } + /** + * @param driverIn the driver to set + */ + public void setDriver(String driverIn) { + this.driver = driverIn; + } + /** + * @return the description + */ + public String getDescription() { + return description; + } + /** + * @param descriptionIn the description to set + */ + public void setDescription(String descriptionIn) { + this.description = descriptionIn; + } + /** + * @return the vendorId + */ + public String getVendorId() { + return vendorId; + } + /** + * @param vendorIdIn the vendorId to set + */ + public void setVendorId(String vendorIdIn) { + this.vendorId = vendorIdIn; + } + /** + * @return the deviceId + */ + public String getDeviceId() { + return deviceId; + } + /** + * @param deviceIdIn the deviceId to set + */ + public void setDeviceId(String deviceIdIn) { + this.deviceId = deviceIdIn; + } + /** + * @return the subVendorId + */ + public String getSubVendorId() { + return subVendorId; + } + /** + * @param subVendorIdIn the subVendorId to set + */ + public void setSubVendorId(String subVendorIdIn) { + this.subVendorId = subVendorIdIn; + } + /** + * @return the subDeviceId + */ + public String getSubDeviceId() { + return subDeviceId; + } + /** + * @param subDeviceIdIn the subDeviceId to set + */ + public void setSubDeviceId(String subDeviceIdIn) { + this.subDeviceId = subDeviceIdIn; + } + /** + * @return the pciType + */ + public long getPciType() { + return pciType; + } + /** + * @param pciTypeIn the pciType to set + */ + public void setPciType(long pciTypeIn) { + this.pciType = pciTypeIn; + } + + + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/RhnPackage.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/RhnPackage.java new file mode 100644 index 0000000..929b381 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/RhnPackage.java @@ -0,0 +1,176 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.db.models; + +/** + * Simple Package DTO + * @version $Rev$ + */ +public class RhnPackage { + + private long id; + private String name; + private String epoch; + private String version; + private String release; + private String arch; + private String description; + private String summary; + + /** + * Getter for pkg id + * @return pkg id + */ + public long getId() { + return id; + } + + /** + * Setter for pkg id + * @param idIn pkg id + */ + public void setId(long idIn) { + this.id = idIn; + } + + /** + * Getter for pkg name + * @return pkg name + */ + public String getName() { + return name; + } + + /** + * Setter for pkg name + * @param nameIn pkg name + */ + public void setName(String nameIn) { + this.name = nameIn; + } + + /** + * Getter for pkg epoch + * @return pkg epoch + */ + public String getEpoch() { + return epoch; + } + + /** + * Setter for pkg epoch + * @param epochIn pkg epoch + */ + public void setEpoch(String epochIn) { + this.epoch = epochIn; + } + + /** + * Getter for pkg version + * @return pkg version + */ + public String getVersion() { + return version; + } + + /** + * Setter for pkg version + * @param versionIn + */ + public void setVersion(String versionIn) { + this.version = versionIn; + } + + /** + * Getter for pkg release + * @return pkg release + */ + public String getRelease() { + return release; + } + + /** + * Setter for pkg release + * @param releaseIn pkg release + */ + public void setRelease(String releaseIn) { + this.release = releaseIn; + } + + /** + * Getter for pkg arch + * @return pkg arch + */ + public String getArch() { + return arch; + } + + /** + * Setter for pkg arch + * @param archIn pkg arch + */ + public void setArch(String archIn) { + this.arch = archIn; + } + + /** + * Setter for pkg description + * @param desc pkg description + */ + public void setDescription(String desc) { + description = desc; + } + + /** + * Getter for pkg description + * @return pkg description + */ + public String getDescription() { + return description; + } + + /** + * Setter for pkg summary + * @param summaryIn pkg summary + */ + public void setSummary(String summaryIn) { + summary = summaryIn; + } + + /** + * Getter for pkg summary + * @return pkg summary + */ + public String getSummary() { + return summary; + } + + /** + * Getter for "pretty" versino + * @return <version>-<release> + */ + public String getPrettyVersion() { + return version + "-" + release; + } + + /** + * Reconstructs filename from various parts of pkg metadata + * @return <name>-<version>-<release>.<arch> + */ + public String getFileName() { + return name + "-" + version + "-" + release + "." + arch; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Server.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Server.java new file mode 100644 index 0000000..4775655 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/Server.java @@ -0,0 +1,698 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.db.models; + +import java.util.Date; + +import org.apache.lucene.document.DateTools; +import org.apache.lucene.document.NumberTools; + + +/** + * Server + * @version $Rev$ + */ +public class Server extends GenericRecord { + + private String name; + private String info; + private String description; + + /* Location */ + private String machine; + private String rack; + private String room; + private String building; + private String address1; + private String address2; + private String city; + private String state; + private String country; + + /* Network Info */ + private String hostname; + private String ipaddr; + + /* DMI Info */ + private String dmiVendor; + private String dmiSystem; + private String dmiProduct; + private String dmiBiosVendor; + private String dmiBiosVersion; + private String dmiBiosRelease; + private String dmiAsset; + private String dmiBoard; + + /* CPU Info */ + private String cpuBogoMIPS; + private String cpuCache; + private String cpuFamily; + private String cpuMhz; + private String cpuStepping; + private String cpuFlags; + private String cpuModel; + private String cpuVersion; + private String cpuVendor; + private String cpuNumberOfCpus; + private String cpuAcpiVersion; + private String cpuApic; + private String cpuApmVersion; + private String cpuChipset; + + private String checkin; + private String registered; + + private String ram; + private String swap; + + private String runningKernel; + + /** + * @return the name + */ + public String getName() { + return name; + } + /** + * @param nameIn the name to set + */ + public void setName(String nameIn) { + this.name = nameIn; + } + + /** + * @return the info + */ + public String getInfo() { + return info; + } + + /** + * @param infoIn the info to set + */ + public void setInfo(String infoIn) { + this.info = infoIn; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param descriptionIn the description to set + */ + public void setDescription(String descriptionIn) { + this.description = descriptionIn; + } + + /** + * @return the machine + */ + public String getMachine() { + return machine; + } + + /** + * @param machineIn the machine to set + */ + public void setMachine(String machineIn) { + this.machine = machineIn; + } + + /** + * @return the rack + */ + public String getRack() { + return rack; + } + + /** + * @param rackIn the rack to set + */ + public void setRack(String rackIn) { + this.rack = rackIn; + } + + /** + * @return the room + */ + public String getRoom() { + return room; + } + + /** + * @param roomIn the room to set + */ + public void setRoom(String roomIn) { + this.room = roomIn; + } + + /** + * @return the building + */ + public String getBuilding() { + return building; + } + + /** + * @param buildingIn the building to set + */ + public void setBuilding(String buildingIn) { + this.building = buildingIn; + } + + /** + * @return the address1 + */ + public String getAddress1() { + return address1; + } + + /** + * @param address1In the address1 to set + */ + public void setAddress1(String address1In) { + this.address1 = address1In; + } + + /** + * @return the address2 + */ + public String getAddress2() { + return address2; + } + + /** + * @param address2In the address2 to set + */ + public void setAddress2(String address2In) { + this.address2 = address2In; + } + + /** + * @return the city + */ + public String getCity() { + return city; + } + + /** + * @param cityIn the city to set + */ + public void setCity(String cityIn) { + this.city = cityIn; + } + + /** + * @return the state + */ + public String getState() { + return state; + } + + /** + * @param stateIn the state to set + */ + public void setState(String stateIn) { + this.state = stateIn; + } + + /** + * @return the country + */ + public String getCountry() { + return country; + } + + /** + * @param countryIn the country to set + */ + public void setCountry(String countryIn) { + this.country = countryIn; + } + + /** + * @return the hostname + */ + public String getHostname() { + return hostname; + } + + /** + * @param hostnameIn the hostname to set + */ + public void setHostname(String hostnameIn) { + this.hostname = hostnameIn; + } + + /** + * @return the ipaddr + */ + public String getIpaddr() { + return ipaddr; + } + + /** + * @param ipaddrIn the ipaddr to set + */ + public void setIpaddr(String ipaddrIn) { + this.ipaddr = ipaddrIn; + } + + /** + * @return the dmiVendor + */ + public String getDmiVendor() { + return dmiVendor; + } + + /** + * @param dmiVendorIn the dmiVendor to set + */ + public void setDmiVendor(String dmiVendorIn) { + this.dmiVendor = dmiVendorIn; + } + + /** + * @return the dmiSystem + */ + public String getDmiSystem() { + return dmiSystem; + } + + /** + * @param dmiSystemIn the dmiSystem to set + */ + public void setDmiSystem(String dmiSystemIn) { + this.dmiSystem = dmiSystemIn; + } + + /** + * @return the dmiProduct + */ + public String getDmiProduct() { + return dmiProduct; + } + + /** + * @param dmiProductIn the dmiProduct to set + */ + public void setDmiProduct(String dmiProductIn) { + this.dmiProduct = dmiProductIn; + } + + /** + * @return the dmiBiosVendor + */ + public String getDmiBiosVendor() { + return dmiBiosVendor; + } + + /** + * @param dmiBiosVendorIn the dmiBiosVendor to set + */ + public void setDmiBiosVendor(String dmiBiosVendorIn) { + this.dmiBiosVendor = dmiBiosVendorIn; + } + + /** + * @return the dmiBiosVersion + */ + public String getDmiBiosVersion() { + return dmiBiosVersion; + } + + /** + * @param dmiBiosVersionIn the dmiBiosVersion to set + */ + public void setDmiBiosVersion(String dmiBiosVersionIn) { + this.dmiBiosVersion = dmiBiosVersionIn; + } + + /** + * @return the dmiBiosRelease + */ + public String getDmiBiosRelease() { + return dmiBiosRelease; + } + + /** + * @param dmiBiosReleaseIn the dmiBiosRelease to set + */ + public void setDmiBiosRelease(String dmiBiosReleaseIn) { + this.dmiBiosRelease = dmiBiosReleaseIn; + } + + /** + * @return the dmiAsset + */ + public String getDmiAsset() { + return dmiAsset; + } + + /** + * @param dmiAssetIn the dmiAsset to set + */ + public void setDmiAsset(String dmiAssetIn) { + this.dmiAsset = dmiAssetIn; + } + + /** + * @return the dmiBoard + */ + public String getDmiBoard() { + return dmiBoard; + } + + /** + * @param dmiBoardIn the dmiBoard to set + */ + public void setDmiBoard(String dmiBoardIn) { + this.dmiBoard = dmiBoardIn; + } + + /** + * @return the cpuBogoMIPS + */ + public String getCpuBogoMIPS() { + return cpuBogoMIPS; + } + + /** + * @param cpuBogoMIPSIn the cpuBogoMIPS to set + */ + public void setCpuBogoMIPS(String cpuBogoMIPSIn) { + if (cpuBogoMIPSIn != null) { + Float f = Float.parseFloat(cpuBogoMIPSIn); + this.cpuBogoMIPS = NumberTools.longToString(f.longValue()); + } + else { + this.cpuBogoMIPS = null; + } + } + + /** + * @return the cpuCache + */ + public String getCpuCache() { + return cpuCache; + } + + /** + * @param cpuCacheIn the cpuCache to set + */ + public void setCpuCache(String cpuCacheIn) { + this.cpuCache = cpuCacheIn; + } + + /** + * @return the cpuFamily + */ + public String getCpuFamily() { + return cpuFamily; + } + + /** + * @param cpuFamilyIn the cpuFamily to set + */ + public void setCpuFamily(String cpuFamilyIn) { + this.cpuFamily = cpuFamilyIn; + } + + /** + * @return the cpuMhz + */ + public String getCpuMhz() { + return cpuMhz; + } + + /** + * @param cpuMhzIn the cpuMhz to set + */ + public void setCpuMhz(String cpuMhzIn) { + if (cpuMhzIn != null) { + this.cpuMhz = NumberTools.longToString(Long.parseLong(cpuMhzIn)); + } + else { + this.cpuMhz = null; + } + } + + /** + * @return the cpuStepping + */ + public String getCpuStepping() { + return cpuStepping; + } + + /** + * @param cpuSteppingIn the cpuStepping to set + */ + public void setCpuStepping(String cpuSteppingIn) { + this.cpuStepping = cpuSteppingIn; + } + + /** + * @return the cpuFlags + */ + public String getCpuFlags() { + return cpuFlags; + } + + /** + * @param cpuFlagsIn the cpuFlags to set + */ + public void setCpuFlags(String cpuFlagsIn) { + this.cpuFlags = cpuFlagsIn; + } + + /** + * @return the cpuModel + */ + public String getCpuModel() { + return cpuModel; + } + + /** + * @param cpuModelIn the cpuModel to set + */ + public void setCpuModel(String cpuModelIn) { + this.cpuModel = cpuModelIn; + } + + /** + * @return the cpuVersion + */ + public String getCpuVersion() { + return cpuVersion; + } + + /** + * @param cpuVersionIn the cpuVersion to set + */ + public void setCpuVersion(String cpuVersionIn) { + this.cpuVersion = cpuVersionIn; + } + + /** + * @return the cpuVendor + */ + public String getCpuVendor() { + return cpuVendor; + } + + /** + * @param cpuVendorIn the cpuVendor to set + */ + public void setCpuVendor(String cpuVendorIn) { + this.cpuVendor = cpuVendorIn; + } + + /** + * @return the cpuNrCpu + */ + public String getCpuNumberOfCpus() { + return cpuNumberOfCpus; + } + + /** + * @param cpuNumberOfCpusIn the cpuNumberOfCpus to set + */ + public void setCpuNumberOfCpus(String cpuNumberOfCpusIn) { + if (cpuNumberOfCpusIn != null) { + this.cpuNumberOfCpus = + NumberTools.longToString(Long.parseLong(cpuNumberOfCpusIn)); + } + else { + this.cpuNumberOfCpus = null; + } + } + + /** + * @return the cpuAcpiVersion + */ + public String getCpuAcpiVersion() { + return cpuAcpiVersion; + } + + /** + * @param cpuAcpiVersionIn the cpuAcpiVersion to set + */ + public void setCpuAcpiVersion(String cpuAcpiVersionIn) { + this.cpuAcpiVersion = cpuAcpiVersionIn; + } + + /** + * @return the cpuApic + */ + public String getCpuApic() { + return cpuApic; + } + + /** + * @param cpuApicIn the cpuApic to set + */ + public void setCpuApic(String cpuApicIn) { + this.cpuApic = cpuApicIn; + } + + /** + * @return the cpuApmVersion + */ + public String getCpuApmVersion() { + return cpuApmVersion; + } + + /** + * @param cpuApmVersionIn the cpuApmVersion to set + */ + public void setCpuApmVersion(String cpuApmVersionIn) { + this.cpuApmVersion = cpuApmVersionIn; + } + + /** + * @return the cpuChipset + */ + public String getCpuChipset() { + return cpuChipset; + } + + /** + * @param cpuChipsetIn the cpuChipset to set + */ + public void setCpuChipset(String cpuChipsetIn) { + this.cpuChipset = cpuChipsetIn; + } + + /** + * @return the checkin + */ + public String getCheckin() { + return checkin; + } + + /** + * @param checkinIn the checkin to set + */ + public void setCheckin(Date checkinIn) { + if (checkinIn != null) { + this.checkin = DateTools.dateToString(checkinIn, + DateTools.Resolution.MINUTE); + } + else { + this.checkin = null; + } + } + + /** + * @return the registered + */ + public String getRegistered() { + return registered; + } + + /** + * @param registeredIn the registered to set + */ + public void setRegistered(Date registeredIn) { + if (registeredIn != null) { + this.registered = DateTools.dateToString(registeredIn, + DateTools.Resolution.MINUTE); + } + else { + this.registered = null; + } + } + + /** + * @return the ram + */ + public String getRam() { + return ram; + } + + /** + * @param ramIn the ram to set + */ + public void setRam(String ramIn) { + if (ramIn != null) { + this.ram = NumberTools.longToString(Long.parseLong(ramIn)); + } + else { + this.ram = null; + } + } + + /** + * @return the swap + */ + public String getSwap() { + return swap; + } + + /** + * @param swapIn the swap to set + */ + public void setSwap(String swapIn) { + if (swapIn != null) { + this.swap = NumberTools.longToString(Long.parseLong(swapIn)); + } + else { + this.swap = null; + } + } + + /** + * @return the running kernel + */ + public String getRunningKernel() { + return runningKernel; + } + + /** + * @param runningKernelIn the runningKernel to set + */ + public void setRunningKernel(String runningKernelIn) { + if (runningKernelIn != null) { + this.runningKernel = runningKernelIn; + } + else { + this.runningKernel = null; + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java new file mode 100644 index 0000000..20937f9 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java @@ -0,0 +1,146 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.db.models; + + +/** + * ServerCustomInfo + * @version $Rev$ + */ +public class ServerCustomInfo extends GenericRecord { + private long keyId; + private long serverId; + private String label; + private String value; + private long createdBy; + private long lastModifiedBy; + private String created; + private String modified; + + /** + * + * @param serverIdIn server id + * @param keyIdIn key id + * @return string based on server id and key id + */ + public static String makeUniqId(long serverIdIn, long keyIdIn) { + return serverIdIn + "-" + keyIdIn; + } + + /** + * + * @return uniqId + */ + public String getUniqId() { + return makeUniqId(serverId, keyId); + } + /** + * @return the keyId + */ + public long getKeyId() { + return keyId; + } + /** + * @param keyIdIn the keyId to set + */ + public void setKeyId(long keyIdIn) { + this.keyId = keyIdIn; + } + /** + * @return the serverId + */ + public long getServerId() { + return serverId; + } + /** + * @param serverIdIn the serverId to set + */ + public void setServerId(long serverIdIn) { + this.serverId = serverIdIn; + } + /** + * @return the created + */ + public String getCreated() { + return created; + } + /** + * @param createdIn the created to set + */ + public void setCreated(String createdIn) { + this.created = createdIn; + } + /** + * @return the modified + */ + public String getModified() { + return modified; + } + /** + * @param modifiedIn the modified to set + */ + public void setModified(String modifiedIn) { + this.modified = modifiedIn; + } + /** + * @return the label + */ + public String getLabel() { + return label; + } + /** + * @param labelIn the label to set + */ + public void setLabel(String labelIn) { + this.label = labelIn; + } + /** + * @return the value + */ + public String getValue() { + return value; + } + /** + * @param valueIn the value to set + */ + public void setValue(String valueIn) { + this.value = valueIn; + } + /** + * @return the createdBy + */ + public long getCreatedBy() { + return createdBy; + } + /** + * @param createdByIn the createdBy to set + */ + public void setCreatedBy(long createdByIn) { + this.createdBy = createdByIn; + } + /** + * @return the lastModifiedBy + */ + public long getLastModifiedBy() { + return lastModifiedBy; + } + /** + * @param lastModifiedByIn the lastModifiedBy to set + */ + public void setLastModifiedBy(long lastModifiedByIn) { + this.lastModifiedBy = lastModifiedByIn; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java new file mode 100644 index 0000000..9ba9178 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java @@ -0,0 +1,128 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.db.models; + + +/** + * SnapshotTag + * @version $Rev$ + */ +public class SnapshotTag extends GenericRecord { + private long id; + private long snapshotId; + private long tagNameId; + private long serverId; + private long orgId; + private String name; + private String created; + private String modified; + /** + * @return the id + */ + public long getId() { + return id; + } + /** + * @param idIn the id to set + */ + public void setId(long idIn) { + this.id = idIn; + } + /** + * @return the snapshotId + */ + public long getSnapshotId() { + return snapshotId; + } + /** + * @param snapshotIdIn the snapshotId to set + */ + public void setSnapshotId(long snapshotIdIn) { + this.snapshotId = snapshotIdIn; + } + /** + * @return the tagNameId + */ + public long getTagNameId() { + return tagNameId; + } + /** + * @param tagNameIdIn the tagNameId to set + */ + public void setTagNameId(long tagNameIdIn) { + this.tagNameId = tagNameIdIn; + } + /** + * @return the serverId + */ + public long getServerId() { + return serverId; + } + /** + * @param serverIdIn the serverId to set + */ + public void setServerId(long serverIdIn) { + this.serverId = serverIdIn; + } + /** + * @return the orgId + */ + public long getOrgId() { + return orgId; + } + /** + * @param orgIdIn the orgId to set + */ + public void setOrgId(long orgIdIn) { + this.orgId = orgIdIn; + } + /** + * @return the name + */ + public String getName() { + return name; + } + /** + * @param nameIn the name to set + */ + public void setName(String nameIn) { + this.name = nameIn; + } + /** + * @return the created + */ + public String getCreated() { + return created; + } + /** + * @param createdIn the created to set + */ + public void setCreated(String createdIn) { + this.created = createdIn; + } + /** + * @return the modified + */ + public String getModified() { + return modified; + } + /** + * @param modifiedIn the modified to set + */ + public void setModified(String modifiedIn) { + this.modified = modifiedIn; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java new file mode 100644 index 0000000..eba617e --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java @@ -0,0 +1,45 @@ +package com.redhat.satellite.search.db.tests; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.Query; +import com.redhat.satellite.search.db.models.RhnPackage; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.List; + +public class DatabaseManagerTest extends BaseTestCase { + + private DatabaseManager dm; + + @Override + protected void setUp() throws Exception { + super.setUp(); + dm = (DatabaseManager) + container.getComponentInstance(DatabaseManager.class); + } + + public void testObjectQuery() throws IOException, SQLException { + Query<Long> maxidquery = dm.getQuery("maxPackageId"); + Long maxid = maxidquery.load(); + Query<RhnPackage> query = dm.getQuery("getPackageById"); + RhnPackage p = query.load(maxid); + assertNotNull(p); + assertEquals(p.getId(), maxid.longValue()); + } + + public void testListQuery() throws IOException, SQLException { + Query<RhnPackage> query = dm.getQuery("listPackagesFromId"); + List<RhnPackage> results = query.loadList((long) 0); + assertNotNull(results); + assertTrue(results.size() > 0); + } + + @SuppressWarnings("unchecked") + @Override + protected Class[] getComponentClasses() { + return TestUtil.buildComponentsList(DatabaseManager.class); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/DocResult.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/DocResult.java new file mode 100644 index 0000000..2e686e7 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/DocResult.java @@ -0,0 +1,119 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + +import org.apache.lucene.document.Document; + + +/** + * DocResult + * @version $Rev$ + */ +public class DocResult extends Result { + + private String url = ""; + private String title = ""; + private String summary = ""; + + /** + * Constructor + */ + public DocResult() { + super(); + } + + /** + * Constructs a result object + * @param rankIn order of results returned from lucene + * @param scoreIn score of this hit as defined by lucene query + * @param doc lucene document containing data fields + */ + public DocResult(int rankIn, float scoreIn, Document doc) { + if (doc.getField("url") != null) { + setUrl(doc.getField("url").stringValue()); + setId(doc.getField("url").stringValue()); + } + if (doc.getField("title") != null) { + setTitle(doc.getField("title").stringValue()); + setName(doc.getField("title").stringValue()); + } + else { + setTitle("EMPTY"); + setName("EMPTY"); + } + setRank(rankIn); + setScore(scoreIn); + } + + /** + * @return the url + */ + public String getUrl() { + return url; + } + + /** + * @param urlIn the url to set + */ + public void setUrl(String urlIn) { + if (urlIn.startsWith("file:")) { + // Translate file:// to a usable URL + String prefix = "/rhn/help"; + int index = urlIn.indexOf(prefix); + if (index > 0) { + urlIn = urlIn.substring(index); + } + } + this.url = urlIn; + } + + /** + * @return the title + */ + public String getTitle() { + return title; + } + + /** + * @param titleIn the title to set + */ + public void setTitle(String titleIn) { + this.title = titleIn; + } + + /** + * @return the summary + */ + public String getSummary() { + return summary; + } + + /** + * @param summaryIn the summary to set + */ + public void setSummary(String summaryIn) { + this.summary = summaryIn; + } + + /** + * @return the string representation of this object + **/ + public String toString() { + return super.toString() + ", Title = " + getTitle() + ", Url = " + getUrl() + + ", Summary = " + getSummary(); + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java new file mode 100644 index 0000000..6bf75e9 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + +import org.apache.lucene.document.Document; + +/** + * Result + * @version $Rev$ + */ +public class HardwareDeviceResult extends Result { + private String serverId; + + /** + * Constructor + */ + public HardwareDeviceResult() { + super(); + serverId = "N/A"; + } + + /** + * Constructs a result object + * @param rankIn order of results returned from lucene + * @param scoreIn score of this hit as defined by lucene query + * @param doc lucene document containing data fields + */ + public HardwareDeviceResult(int rankIn, float scoreIn, Document doc) { + if (doc.getField("description") != null) { + setName(doc.getField("description").stringValue()); + } + if (doc.getField("serverId") != null) { + setServerId(doc.getField("serverId").stringValue()); + } + if (doc.getField("id") != null) { + setId(doc.getField("id").stringValue()); + } + setRank(rankIn); + setScore(scoreIn); + } + + + /** + * @return the serverId + */ + public String getServerId() { + return serverId; + } + + /** + * @param serverIdIn the serverId to set + */ + public void setServerId(String serverIdIn) { + this.serverId = serverIdIn; + } + + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexManager.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexManager.java new file mode 100644 index 0000000..23e4f5a --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexManager.java @@ -0,0 +1,837 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + +import com.redhat.satellite.search.config.Configuration; +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.ngram.NGramAnalyzer; +import com.redhat.satellite.search.index.ngram.NGramQueryParser; +import com.redhat.satellite.search.rpc.handlers.IndexHandler; + +import org.apache.hadoop.fs.FileSystem; + +import org.apache.log4j.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; +import org.apache.lucene.document.DateTools; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.LockObtainFailedException; + +import org.apache.nutch.analysis.AnalyzerFactory; +import org.apache.nutch.searcher.FetchedSegments; +import org.apache.nutch.searcher.HitDetails; +import org.apache.nutch.searcher.Summary; + +import org.apache.nutch.util.NutchConfiguration; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Indexing workhorse class + * + * @version $Rev$ + */ +public class IndexManager { + + private static Logger log = Logger.getLogger(IndexManager.class); + private String indexWorkDir; + private int maxHits; + private double score_threshold; + private double system_score_threshold; + private double errata_score_threshold; + private double errata_advisory_score_threshold; + private int min_ngram; + private int max_ngram; + private boolean filterDocResults = false; + private boolean explainResults = false; + private AnalyzerFactory nutchAnalyzerFactory; + // Name conflict with our Configuration class and Hadoop's + private org.apache.hadoop.conf.Configuration nutchConf; + private Map<String, String> docLocaleLookUp = new HashMap<String, String>(); + private Map<String, FetchedSegments> docSegments; + /** + * Constructor + * + * @param config application config + */ + public IndexManager(Configuration config) { + maxHits = config.getInt("search.max_hits_returned", 50); + indexWorkDir = config.getString("search.index_work_dir", null); + if (indexWorkDir == null) { + throw new IllegalArgumentException( + "search.index_work_dir config entry " + "is missing"); + } + if (!indexWorkDir.endsWith("/")) { + indexWorkDir += "/"; + } + score_threshold = config.getDouble("search.score_threshold", .30); + system_score_threshold = config.getDouble("search.system_score_threshold", .30); + errata_score_threshold = config.getDouble("search.errata_score_threshold", .30); + errata_advisory_score_threshold = + config.getDouble("search.errata.advisory_score_threshold", .30); + min_ngram = config.getInt("search.min_ngram", 1); + max_ngram = config.getInt("search.max_ngram", 5); + initDocLocaleLookup(); + filterDocResults = config.getBoolean("search.doc.limit_results"); + explainResults = config.getBoolean("search.log.explain.results"); + initDocSummary(); + } + + + /** + * @return String of the index working directory + */ + public String getIndexWorkDir() { + return indexWorkDir; + } + + /** + * Query a index + * + * @param indexName name of the index + * @param query search query + * @param lang language + * @return list of hits + * @throws IndexingException if there is a problem indexing the content. + * @throws QueryParseException + */ + public List<Result> search(String indexName, String query, String lang) + throws IndexingException, QueryParseException { + return search(indexName, query, lang, false); + } + + /** + * Query a index + * + * @param indexName name of the index + * @param query search query + * @param lang language + * @param isFineGrained + * true: will limit results, less are returned but they are closer + * to the search query, useful for advanced/free form queries + * + * false: will allow queries to be more flexible returning words + * which are spelled similarly + * + * @return list of hits + * @throws IndexingException if there is a problem indexing the content. + * @throws QueryParseException + */ + public List<Result> search(String indexName, String query, String lang, + boolean isFineGrained) + throws IndexingException, QueryParseException { + IndexSearcher searcher = null; + IndexReader reader = null; + List<Result> retval = null; + try { + reader = getIndexReader(indexName, lang); + searcher = getIndexSearcher(indexName, lang); + QueryParser qp = getQueryParser(indexName, lang, isFineGrained); + Query q = qp.parse(query); + if (log.isDebugEnabled()) { + log.debug("Original query was: " + query); + log.debug("Parsed Query is: " + q.toString()); + } + Hits hits = searcher.search(q); + if (log.isDebugEnabled()) { + log.debug(hits.length() + " results were found."); + } + Set<Term> queryTerms = null; + try { + queryTerms = new HashSet<Term>(); + Query newQ = q.rewrite(reader); + newQ.extractTerms(queryTerms); + } + catch (Exception e) { + e.printStackTrace(); + throw new QueryParseException(e); + } + retval = processHits(indexName, hits, queryTerms, query, lang); + if (explainResults) { + debugExplainResults(indexName, hits, searcher, q, queryTerms); + } + } + catch (IOException e) { + throw new IndexingException(e); + } + catch (ParseException e) { + throw new QueryParseException("Could not parse query: '" + query + "'"); + } + finally { + try { + if (searcher != null) { + searcher.close(); + } + if (reader != null) { + reader.close(); + } + } + catch (IOException ex) { + throw new IndexingException(ex); + } + } + return retval; + } + + /** + * Adds a document to an index + * + * @param indexName index to use + * @param doc Document to be indexed. + * @param lang language. + * @throws IndexingException something went wrong adding the document + */ + public void addToIndex(String indexName, Document doc, String lang) + throws IndexingException { + + try { + IndexWriter writer = getIndexWriter(indexName, lang); + try { + writer.addDocument(doc); + writer.flush(); + } + finally { + try { + writer.close(); + } + finally { + // unlock it if it is locked. + unlockIndex(indexName); + } + } + } + catch (CorruptIndexException e) { + throw new IndexingException(e); + } + catch (LockObtainFailedException e) { + throw new IndexingException(e); + } + catch (IOException e) { + throw new IndexingException(e); + } + } + /** + * @param indexName + * @param doc document with data to index + * @param uniqueField field in doc which identifies this uniquely + * @param lang language + * @throws IndexingException + */ + public void addUniqueToIndex(String indexName, Document doc, + String uniqueField, String lang) + throws IndexingException { + IndexReader reader = null; + int numFound = 0; + try { + reader = getIndexReader(indexName, lang); + Term term = new Term(uniqueField, doc.get(uniqueField)); + numFound = reader.docFreq(term); + } + catch (FileNotFoundException e) { + // Index doesn't exist, so this add will be unique + // we don't need to do anything/ + } + catch (IOException e) { + throw new IndexingException(e); + } + finally { + if (reader != null) { + try { + reader.close(); + } + catch (IOException e) { + // + } + } + } + if (numFound > 0) { + log.info("Found " + numFound + " <" + indexName + " docs for " + + uniqueField + ":" + doc.get(uniqueField) + + " will remove them now."); + removeFromIndex(indexName, uniqueField, doc.get(uniqueField)); + } + addToIndex(indexName, doc, lang); + } + + /** + * Remove a document from an index + * + * @param indexName index to use + * @param uniqueField field name which represents this data's unique id + * @param objectId unique document id + * @throws IndexingException something went wrong removing the document + */ + public void removeFromIndex(String indexName, String uniqueField, String objectId) + throws IndexingException { + log.info("Removing <" + indexName + "> " + uniqueField + ":" + + objectId); + Term t = new Term(uniqueField, objectId); + IndexReader reader; + try { + reader = getIndexReader(indexName, IndexHandler.DEFAULT_LANG); + try { + reader.deleteDocuments(t); + reader.flush(); + } + finally { + if (reader != null) { + reader.close(); + } + } + } + catch (CorruptIndexException e) { + throw new IndexingException(e); + } + catch (IOException e) { + throw new IndexingException(e); + } + } + + /** + * Unlocks the index at the given directory if it is currently locked. + * Otherwise, does nothing. + * @param indexName index name + * @throws IOException thrown if there is a problem unlocking the index. + */ + private void unlockIndex(String indexName) throws IOException { + String path = indexWorkDir + indexName; + File f = new File(path); + Directory dir = FSDirectory.getDirectory(f); + if (IndexReader.isLocked(dir)) { + IndexReader.unlock(dir); + } + } + + private IndexWriter getIndexWriter(String name, String lang) + throws CorruptIndexException, LockObtainFailedException, + IOException { + String path = indexWorkDir + name; + File f = new File(path); + f.mkdirs(); + Analyzer analyzer = getAnalyzer(name, lang); + IndexWriter writer = new IndexWriter(path, analyzer); + writer.setUseCompoundFile(true); + return writer; + } + + private IndexReader getIndexReader(String indexName, String locale) + throws CorruptIndexException, IOException { + String path = ""; + if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { + path = indexWorkDir + File.separator + + getDocIndexPath(locale); + } + else { + path = indexWorkDir + indexName; + } + log.info("IndexManager::getIndexReader(" + indexName + ", " + locale + + ") path = " + path); + File f = new File(path); + IndexReader retval = IndexReader.open(FSDirectory.getDirectory(f)); + return retval; + } + + private IndexSearcher getIndexSearcher(String indexName, String locale) + throws CorruptIndexException, IOException { + String path = ""; + if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { + path = indexWorkDir + File.separator + + getDocIndexPath(locale); + } + else { + path = indexWorkDir + indexName; + } + log.info("IndexManager::getIndexSearcher(" + indexName + ", " + locale + + ") path = " + path); + IndexSearcher retval = new IndexSearcher(path); + return retval; + } + + private QueryParser getQueryParser(String indexName, String lang, + boolean isFineGrained) { + if (log.isDebugEnabled()) { + log.debug("getQueryParser(" + indexName + ", " + lang + ", " + + isFineGrained + ")"); + } + QueryParser qp; + Analyzer analyzer = getAnalyzer(indexName, lang); + if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { + qp = new QueryParser("content", analyzer); + } + else { + qp = new NGramQueryParser("name", analyzer, isFineGrained); + } + qp.setDateResolution(DateTools.Resolution.MINUTE); + return qp; + } + + + private Analyzer getAnalyzer(String indexName, String lang) { + if (log.isDebugEnabled()) { + log.debug("getAnalyzer(" + indexName + ", " + lang + ")"); + } + if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { + return getDocAnalyzer(lang); + } + else if (indexName.compareTo(BuilderFactory.SERVER_TYPE) == 0) { + return getServerAnalyzer(); + } + else if (indexName.compareTo(BuilderFactory.SNAPSHOT_TAG_TYPE) == 0) { + return getSnapshotTagAnalyzer(); + } + else if (indexName.compareTo(BuilderFactory.HARDWARE_DEVICE_TYPE) == 0) { + return getHardwareDeviceAnalyzer(); + } + else if (indexName.compareTo(BuilderFactory.SERVER_CUSTOM_INFO_TYPE) == 0) { + return getServerCustomInfoAnalyzer(); + } + else { + log.debug(indexName + " using getDefaultAnalyzer()"); + return getDefaultAnalyzer(); + } + } + + private List<Result> processHits(String indexName, Hits hits, Set<Term> queryTerms, + String query, String lang) + throws IOException { + List<Result> retval = new ArrayList<Result>(); + for (int x = 0; x < hits.length(); x++) { + Document doc = hits.doc(x); + Result pr = null; + if (!isScoreAcceptable(indexName, hits, x, query)) { + break; + } + if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { + pr = new DocResult(x, hits.score(x), doc); + String summary = lookupDocSummary(doc, query, lang); + if (summary != null) { + ((DocResult)pr).setSummary(summary); + } + } + else if (indexName.compareTo(BuilderFactory.HARDWARE_DEVICE_TYPE) == 0) { + pr = new HardwareDeviceResult(x, hits.score(x), doc); + } + else if (indexName.compareTo(BuilderFactory.SNAPSHOT_TAG_TYPE) == 0) { + pr = new SnapshotTagResult(x, hits.score(x), doc); + } + else if (indexName.compareTo(BuilderFactory.SERVER_CUSTOM_INFO_TYPE) == 0) { + pr = new ServerCustomInfoResult(x, hits.score(x), doc); + } + else { + pr = new Result(x, + doc.getField("id").stringValue(), + doc.getField("name").stringValue(), + hits.score(x)); + } + if (log.isDebugEnabled()) { + log.debug("Hit[" + x + "] Score = " + hits.score(x) + ", Result = " + pr); + } + /** + * matchingField will help the webUI to understand what field was responsible + * for this match. Later implementation should use "Explanation" to determine + * field, for now we will simply grab one term and return it's field. + */ + try { + MatchingField match = new MatchingField(query, doc, queryTerms); + pr.setMatchingField(match.getFieldName()); + pr.setMatchingFieldValue(match.getFieldValue()); + log.info("hit[" + x + "] matchingField is being set to: <" + + pr.getMatchingField() + "> based on passed in query field. " + + "matchingFieldValue = " + pr.getMatchingFieldValue()); + } + catch (Exception e) { + log.error("Caught exception: ", e); + } + if (pr != null) { + retval.add(pr); + } + if (x == maxHits) { + break; + } + } + return retval; + } + /** + * + * @param indexName + * @param hits + * @param x + * @param query + * @return true - score is acceptable + * false - score is NOT acceptable + * @throws IOException + */ + private boolean isScoreAcceptable(String indexName, Hits hits, int x, String queryIn) + throws IOException { + String guessMainQueryTerm = MatchingField.getFirstFieldName(queryIn); + + if ((indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) && + (!filterDocResults)) { + return true; + } + /** + * Dropping matches which are a poor fit. + * system searches are filtered based on "system_score_threshold" + * other searches will return 10 best matches, then filter anything below + * "score_threshold" + */ + if ((indexName.compareTo(BuilderFactory.SERVER_TYPE) == 0) || + (indexName.compareTo(BuilderFactory.SERVER_CUSTOM_INFO_TYPE) == 0) || + (indexName.compareTo(BuilderFactory.SNAPSHOT_TAG_TYPE) == 0) || + (indexName.compareTo(BuilderFactory.HARDWARE_DEVICE_TYPE) == 0)) { + if (hits.score(x) < system_score_threshold) { + if (log.isDebugEnabled()) { + log.debug("hits.score(" + x + ") is " + hits.score(x)); + log.debug("Filtering out search results from " + x + " to " + + hits.length() + ", due to their score being below " + + "system_score_threshold = " + system_score_threshold); + } + return false; + } + } + else if (indexName.compareTo(BuilderFactory.ERRATA_TYPE) == 0) { + if (guessMainQueryTerm.compareTo("advisoryName") == 0) { + if (hits.score(x) < errata_advisory_score_threshold) { + if (log.isDebugEnabled()) { + log.debug("hits.score(" + x + ") is " + hits.score(x)); + log.debug("Filtering out search results from " + x + " to " + + hits.length() + ", due to their score being below " + + "errata_advisory_score_threshold = " + + errata_advisory_score_threshold); + } + return false; + } + } + else { + if (hits.score(x) < errata_score_threshold) { + if (log.isDebugEnabled()) { + log.debug("hits.score(" + x + ") is " + hits.score(x)); + log.debug("Filtering out search results from " + x + " to " + + hits.length() + ", due to their score being below " + + "errata_score_threshold = " + + errata_score_threshold); + } + return false; + } + } + } + else if (((hits.score(x) < score_threshold) && (x > 10)) || + (hits.score(x) < 0.001)) { + /** + * Dropping matches which are a poor fit. + * First term is configurable, it allows matches like spelling errors or + * suggestions to be possible. + * Second term is intended to get rid of pure and utter crap hits + */ + if (log.isDebugEnabled()) { + log.debug("hits.score(" + x + ") is " + hits.score(x)); + log.debug("Filtering out search results from " + x + " to " + + hits.length() + ", due to their score being below " + + "score_threshold = " + score_threshold); + } + return false; + } + return true; + } + + /** + * Removes any documents which are not related to the passed in Set of good value + * @param ids Set of ids of all known/good values + * @param indexName index name to operate on + * @param uniqField the name of the field in the Document to uniquely identify + * this record + * @return the number of documents deleted + */ + public int deleteRecordsNotInList(Set<String> ids, String indexName, + String uniqField) { + int count = 0; + IndexReader reader = null; + try { + reader = getIndexReader(indexName, IndexHandler.DEFAULT_LANG); + int numDocs = reader.numDocs(); + for (int i = 0; i < numDocs; i++) { + if (!reader.isDeleted(i)) { + Document doc = reader.document(i); + String uniqId = doc.getField(uniqField).stringValue(); + if (!ids.contains(uniqId)) { + log.info(indexName + ":" + uniqField + ": <" + uniqId + + "> not found in list of current/good values " + + "assuming this has been deleted from Database and we " + + "should remove it."); + removeFromIndex(indexName, uniqField, uniqId); + count++; + } + } + } + } + catch (IOException e) { + e.printStackTrace(); + log.info("deleteRecordsNotInList() caught exception : " + e); + } + catch (IndexingException e) { + e.printStackTrace(); + log.info("deleteRecordsNotInList() caught exception : " + e); + } + finally { + if (reader != null) { + try { + reader.close(); + } + catch (IOException e) { + // + } + } + } + return count; + } + + private void debugExplainResults(String indexName, Hits hits, IndexSearcher searcher, + Query q, Set<Term> queryTerms) + throws IOException { + log.debug("Parsed Query is " + q.toString()); + log.debug("Looking at index: " + indexName); + for (int i = 0; i < hits.length(); i++) { + if ((i < 10)) { + Document doc = hits.doc(i); + Float score = hits.score(i); + Explanation ex = searcher.explain(q, hits.id(i)); + log.debug("Looking at hit<" + i + ", " + hits.id(i) + ", " + score + + ">: " + doc); + log.debug("Explanation: " + ex); + MatchingField match = new MatchingField(q.toString(), doc, queryTerms); + String fieldName = match.getFieldName(); + String fieldValue = match.getFieldValue(); + log.debug("Guessing that matched fieldName is " + fieldName + " = " + + fieldValue); + } + } + } + + private String getDocIndexPath(String lang) throws IOException { + if (docLocaleLookUp.containsKey(lang)) { + return BuilderFactory.DOCS_TYPE + File.separator + + docLocaleLookUp.get(lang); + } + log.error("Unable to find docs index dir for language " + lang); + throw new IOException("Unable to find docs index dir for language: " + lang); + } + + + private Analyzer getDocAnalyzer(String lang) { + /** + * We want to use the same Analyzer nutch is using when the indexes are + * generated + * */ + Analyzer analyzer = null; + try { + analyzer = nutchAnalyzerFactory.get(lang); + } + catch (Exception e) { + log.info("Caught exception, nutch is most likely not installed"); + log.info("Defaulting to generic analyzer for Documentation Search"); + log.info("Install nutch package to get summary info and better matches."); + analyzer = new StandardAnalyzer(); + } + log.info("Language choice is " + lang + ", analyzer chosen is " + + analyzer); + return analyzer; + } + + private Analyzer getServerAnalyzer() { + PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new + NGramAnalyzer(min_ngram, max_ngram)); + analyzer.addAnalyzer("checkin", new KeywordAnalyzer()); + analyzer.addAnalyzer("registered", new KeywordAnalyzer()); + analyzer.addAnalyzer("ram", new KeywordAnalyzer()); + analyzer.addAnalyzer("swap", new KeywordAnalyzer()); + analyzer.addAnalyzer("cpuMhz", new KeywordAnalyzer()); + analyzer.addAnalyzer("cpuNumberOfCpus", new KeywordAnalyzer()); + + + return analyzer; + } + + private Analyzer getSnapshotTagAnalyzer() { + PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new + NGramAnalyzer(min_ngram, max_ngram)); + analyzer.addAnalyzer("id", new KeywordAnalyzer()); + analyzer.addAnalyzer("snapshotId", new KeywordAnalyzer()); + analyzer.addAnalyzer("orgId", new KeywordAnalyzer()); + analyzer.addAnalyzer("serverId", new KeywordAnalyzer()); + analyzer.addAnalyzer("tagNameId", new KeywordAnalyzer()); + analyzer.addAnalyzer("created", new KeywordAnalyzer()); + analyzer.addAnalyzer("modified", new KeywordAnalyzer()); + return analyzer; + } + + private Analyzer getHardwareDeviceAnalyzer() { + PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new + NGramAnalyzer(min_ngram, max_ngram)); + analyzer.addAnalyzer("id", new KeywordAnalyzer()); + analyzer.addAnalyzer("serverId", new KeywordAnalyzer()); + analyzer.addAnalyzer("pciType", new KeywordAnalyzer()); + return analyzer; + } + + private Analyzer getServerCustomInfoAnalyzer() { + PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new + NGramAnalyzer(min_ngram, max_ngram)); + analyzer.addAnalyzer("id", new KeywordAnalyzer()); + analyzer.addAnalyzer("serverId", new KeywordAnalyzer()); + analyzer.addAnalyzer("created", new KeywordAnalyzer()); + analyzer.addAnalyzer("modified", new KeywordAnalyzer()); + analyzer.addAnalyzer("createdBy", new KeywordAnalyzer()); + analyzer.addAnalyzer("lastModifiedBy", new KeywordAnalyzer()); + return analyzer; + } + + private Analyzer getDefaultAnalyzer() { + PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new + NGramAnalyzer(min_ngram, max_ngram)); + analyzer.addAnalyzer("id", new KeywordAnalyzer()); + analyzer.addAnalyzer("arch", new KeywordAnalyzer()); + analyzer.addAnalyzer("epoch", new KeywordAnalyzer()); + analyzer.addAnalyzer("version", new KeywordAnalyzer()); + analyzer.addAnalyzer("release", new KeywordAnalyzer()); + analyzer.addAnalyzer("filename", new KeywordAnalyzer()); + return analyzer; + } + + private boolean initDocSummary() { + /** + * NOTE: NutchConfiguration is expecting "nutch-default.xml" and "nutch-site.xml" + * to be available in the CLASSPATH + */ + try { + nutchConf = NutchConfiguration.create(); + nutchAnalyzerFactory = new AnalyzerFactory(nutchConf); + FileSystem fs = FileSystem.get(nutchConf); + docSegments = new HashMap<String, FetchedSegments>(); + for (String key : docLocaleLookUp.keySet()) { + String lang = docLocaleLookUp.get(key); + String segmentsDir = indexWorkDir + File.separator + + getDocIndexPath(key) + File.separator + "segments"; + FetchedSegments segments = new FetchedSegments(fs, segmentsDir, nutchConf); + if (segments == null) { + log.info("Unable to create docSegments for language: " + key); + docSegments.put(key, null); + } + String[] segNames = segments.getSegmentNames(); + if (segNames == null || segNames.length == 0) { + log.info("Unable to find any segments for language: " + key); + docSegments.put(key, null); + } + log.info("Adding Documentation segments for language: " + key); + docSegments.put(key, segments); + } + } + catch (Exception e) { + log.error("ignoring exception - most likely Nutch isn't present, so" + + " doc summaries will be empty"); + e.printStackTrace(); + } + return true; + } + + private String lookupDocSummary(Document doc, String queryString, String lang) { + if (docSegments == null) { + log.info("docSegments is null, doc summary not possible"); + log.info("nutch is probably not installed, install nutch to get summary info"); + return ""; + } + if (!docSegments.containsKey(lang)) { + log.info("Couldn't find segments info for " + lang); + log.info("Summary info will be missing for " + lang); + return ""; + } + FetchedSegments segments = docSegments.get(lang); + if (segments == null) { + log.info("Segments info for " + lang + " is null"); + return ""; + } + try { + if (log.isDebugEnabled()) { + log.debug("Attempting lookupDocSummary<" + lang + "> for " + doc); + } + HitDetails hd = new HitDetails(doc.getField("segment").stringValue(), + doc.getField("url").stringValue()); + // NOTE: Name conflict with Nutch's Query versus Lucene Query + org.apache.nutch.searcher.Query query = + org.apache.nutch.searcher.Query.parse(queryString, nutchConf); + Summary sum = segments.getSummary(hd, query); + if (log.isDebugEnabled()) { + log.debug("Will return summary<" + lang + "> = " + sum.toString()); + } + return sum.toString(); + } + catch (Exception e) { + log.info("Failed to lookupDocSummary<" + lang + ">, caught Exception: " + e); + e.printStackTrace(); + } + return ""; + } + + private void initDocLocaleLookup() { + docLocaleLookUp.put("bn_IN", "bn-IN"); + docLocaleLookUp.put("de", "de-DE"); + docLocaleLookUp.put("en_US", "en-US"); + docLocaleLookUp.put("es", "es-ES"); + docLocaleLookUp.put("fr", "fr-FR"); + docLocaleLookUp.put("gu", "gu-IN"); + docLocaleLookUp.put("hi", "hi-IN"); + docLocaleLookUp.put("it", "it-IT"); + docLocaleLookUp.put("ja", "ja-JP"); + docLocaleLookUp.put("ko", "ko-KR"); + docLocaleLookUp.put("pa", "pa-IN"); + docLocaleLookUp.put("pt_BR", "pt-BR"); + docLocaleLookUp.put("ru", "ru-RU"); + docLocaleLookUp.put("ta", "ta-IN"); + docLocaleLookUp.put("zh_CN", "zh-CN"); + docLocaleLookUp.put("zh_TW", "zh-TW"); + // Below exist in docs, but weren't available as a doc option from + // satellite webui and they weren't available Locales on my machine + // guessing at what they will look at. + docLocaleLookUp.put("as", "as-IN"); + docLocaleLookUp.put("ml", "ml-IN"); + docLocaleLookUp.put("mr", "mr-IN"); + docLocaleLookUp.put("or", "or-IN"); + docLocaleLookUp.put("kn", "kn-IN"); + docLocaleLookUp.put("si_lk", "si-LK"); + docLocaleLookUp.put("te", "te-IN"); + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexingException.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexingException.java new file mode 100644 index 0000000..167de6e --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/IndexingException.java @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + +/** + * Something vewy vewy bad happened while indexing content + * + * @version $Rev$ + */ +public class IndexingException extends Exception { + + private static final long serialVersionUID = 3037442150973499615L; + + /** + * Simple no-arg constructor + */ + public IndexingException() { + super(); + } + + /** + * Constructor + * + * @param message + * error message + * @param cause + * root cause + */ + public IndexingException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor + * + * @param message + * error message + */ + public IndexingException(String message) { + super(message); + } + + /** + * Constructor + * + * @param cause + * root cause + */ + public IndexingException(Throwable cause) { + super(cause); + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java new file mode 100644 index 0000000..06dd221 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index; + +import java.io.Reader; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.CharTokenizer; + + + +/** + * KeywordAnalyzer + * Used to retain all characters associated with a search term. Basically + * use this if you don't want the search term to be tokenized. + * + * @version $Rev$ + */ +public class KeywordAnalyzer extends Analyzer { + /** + * Will return the exact test passed in. + *@param fieldName ignored value + *@param reader Reader containing data to parse + *@return TokenStream which contains exact text passed in. + */ + @Override + public TokenStream tokenStream(String fieldName, Reader reader) { + return new CharTokenizer(reader) { + protected boolean isTokenChar(char c) { + return true; + } + }; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/MatchingField.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/MatchingField.java new file mode 100644 index 0000000..8c25e88 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/MatchingField.java @@ -0,0 +1,119 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + +import org.apache.log4j.Logger; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumberTools; +import org.apache.lucene.index.Term; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; + +/** + * Purpose of this class is to guess what field in the Document is responsible + * for this being flagged as a hit. We will use the query and terms as hints. + * @version $Rev$ + */ +public class MatchingField { + private static Logger log = Logger.getLogger(IndexManager.class); + protected Document doc; + protected Object[] terms; + protected String query; + protected Map needNumberToolsAdjust; + + /** + * + * @param queryIn query used in search + * @param docIn document results returned as a match + * @param termsIn terms from a parsed query + */ + public MatchingField(String queryIn, Document docIn, Set<Term> termsIn) { + query = queryIn; + doc = docIn; + if ((termsIn != null)) { + terms = termsIn.toArray(); + } + else { + terms = new Term[0]; + } + needNumberToolsAdjust = new HashMap(); + needNumberToolsAdjust.put("cpuMhz", true); + needNumberToolsAdjust.put("cpuBogoMIPS", true); + needNumberToolsAdjust.put("cpuMhz", true); + needNumberToolsAdjust.put("cpuNumberOfCpus", true); + needNumberToolsAdjust.put("ram", true); + needNumberToolsAdjust.put("swap", true); + } + + /** + * + * @return field name most responsible for this document being a match + */ + public String getFieldName() { + if (terms.length > 0) { + return ((Term)terms[0]).field(); + } + return getFirstFieldName(query); + } + + /** + * + * @return value most responsible for this document being a match + */ + public String getFieldValue() { + String fieldName = getFieldName(); + Field f = doc.getField(fieldName); + if (f == null) { + StringBuffer sb = new StringBuffer(); + sb.append("[length=" + terms.length + "; "); + for (Object o : terms) { + sb.append(o + ", "); + } + sb.append("]"); + log.info("Unable to get matchingFieldValue for field : " + fieldName + + " with query: " + query + ", and terms = " + sb.toString()); + log.info("Document = " + doc); + return ""; + } + String value = f.stringValue(); + if (needNumberToolsAdjust.containsKey(fieldName)) { + Long temp = NumberTools.stringToLong(value); + value = temp.toString(); + } + return value; + } + + /** + * + * @param queryIn + * @return first term in query, which is a good guess as to being the most + * important term in the query. + * + * */ + public static String getFirstFieldName(String queryIn) { + // remove first parenthesis, this shows up when you do a + // Query.toString(), package searches generally have this + if (queryIn.startsWith("(")) { + queryIn = queryIn.substring(1); + } + StringTokenizer tokens = new StringTokenizer(queryIn, ":"); + return tokens.nextToken(); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/QueryParseException.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/QueryParseException.java new file mode 100644 index 0000000..e4b0744 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/QueryParseException.java @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index; + + +/** + * QueryParseException thrown if there is a problem parsing the query input. + * @version $Rev$ + */ +public class QueryParseException extends Exception { + /** + * Simple no-arg constructor + */ + public QueryParseException() { + super(); + } + + /** + * Constructor + * @param message error message + * @param cause root cause + */ + public QueryParseException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor + * @param message error message + */ + public QueryParseException(String message) { + super(message); + } + + /** + * Constructor + * @param cause root cause + */ + public QueryParseException(Throwable cause) { + super(cause); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/Result.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/Result.java new file mode 100644 index 0000000..793c54d --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/Result.java @@ -0,0 +1,152 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + + +/** + * PackageResult + * @version $Rev$ + */ +public class Result { + private int rank = 0; + private String id = ""; + private String name = ""; + private float score = 0.0f; + + /** TODO: + * matchingField could be used to possibly give more info into what made + * lucene think this was a good match.. + */ + private String matchingField = ""; + private String matchingFieldValue = ""; + /** + * Constructor + */ + public Result() { + rank = -1; + id = ""; + name = ""; + score = 0; + } + + /** + * Constructs a pre populated + * @param r rank + * @param idIn package id + * @param nameIn package name + */ + public Result(int r, String idIn, String nameIn, float scoreIn) { + rank = r; + id = idIn; + name = nameIn; + score = scoreIn; + } + + /** + * Sets the rank + * @param rankIn rank + */ + public void setRank(int rankIn) { + rank = rankIn; + } + /** + * Returns the rank. + * @return the rank. + */ + public int getRank() { + return rank; + } + + /** + * Sets the id + * @param idIn id + */ + public void setId(String idIn) { + id = idIn; + } + + /** + * Returns the package id. + * @return the package id. + */ + public String getId() { + return id; + } + + /** + * Sets the name + * @param nameIn name + */ + public void setName(String nameIn) { + name = nameIn; + } + /** + * Returns the package name. + * @return the package name. + */ + public String getName() { + return name; + } + + /** + * Sets the score from lucene + * @param scoreIn score + */ + public void setScore(float scoreIn) { + score = scoreIn; + } + /** + * Returns the score. + * @return the score. + */ + public float getScore() { + return score; + } + + /** + * @return the matchingField + */ + public String getMatchingField() { + return matchingField; + } + + /** + * @param matchingFieldIn the matchingField to set + */ + public void setMatchingField(String matchingFieldIn) { + this.matchingField = matchingFieldIn; + } + /** + * @return the matchingFieldValue + */ + public String getMatchingFieldValue() { + return matchingFieldValue; + } + /** + * @param matchingFieldValueIn the matchingFieldValue to set + */ + public void setMatchingFieldValue(String matchingFieldValueIn) { + this.matchingFieldValue = matchingFieldValueIn; + } + /** + * @return the string representation of this object + **/ + public String toString() { + return "Name = " + getName() + ", Id = " + getId() + ", Score = " + getScore() + + ", Rank = " + getRank(); + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java new file mode 100644 index 0000000..06e45bd --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java @@ -0,0 +1,86 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + +import org.apache.lucene.document.Document; + +/** + * ServerCustomInfoResult + * @version $Rev$ + */ +public class ServerCustomInfoResult extends Result { + private String serverId; + private String value; + /** + * Constructor + */ + public ServerCustomInfoResult() { + super(); + serverId = "N/A"; + value = "N/A"; + } + + /** + * Constructs a result object + * @param rankIn order of results returned from lucene + * @param scoreIn score of this hit as defined by lucene query + * @param doc lucene document containing data fields + */ + public ServerCustomInfoResult(int rankIn, float scoreIn, Document doc) { + if (doc.getField("value") != null) { + setValue(doc.getField("value").stringValue()); + } + if (doc.getField("serverId") != null) { + setServerId(doc.getField("serverId").stringValue()); + } + if (doc.getField("id") != null) { + setId(doc.getField("id").stringValue()); + } + setRank(rankIn); + setScore(scoreIn); + } + + + /** + * @return the serverId + */ + public String getServerId() { + return serverId; + } + + /** + * @param serverIdIn the serverId to set + */ + public void setServerId(String serverIdIn) { + this.serverId = serverIdIn; + } + + /** + * @return the value + */ + public String getValue() { + return value; + } + + /** + * @param valueIn the value to set + */ + public void setValue(String valueIn) { + this.value = valueIn; + } + + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java new file mode 100644 index 0000000..cedea5b --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index; + +import org.apache.lucene.document.Document; + +/** + * SnapshotTagResult + * @version $Rev$ + */ +public class SnapshotTagResult extends Result { + private String serverId; + private String snapshotId; + + /** + * Constructor + */ + public SnapshotTagResult() { + super(); + serverId = "N/A"; + } + + /** + * Constructs a result object + * @param rankIn order of results returned from lucene + * @param scoreIn score of this hit as defined by lucene query + * @param doc lucene document containing data fields + */ + public SnapshotTagResult(int rankIn, float scoreIn, Document doc) { + if (doc.getField("id") != null) { + setId(doc.getField("id").stringValue()); + } + if (doc.getField("name") != null) { + setName(doc.getField("name").stringValue()); + } + if (doc.getField("serverId") != null) { + setServerId(doc.getField("serverId").stringValue()); + } + if (doc.getField("snapshotId") != null) { + setSnapshotId(doc.getField("snapshotId").stringValue()); + } + setRank(rankIn); + setScore(scoreIn); + } + + + /** + * @return the serverId + */ + public String getServerId() { + return serverId; + } + + /** + * @param serverIdIn the serverId to set + */ + public void setServerId(String serverIdIn) { + this.serverId = serverIdIn; + } + + /** + * @return the snapshotId + */ + public String getSnapshotId() { + return snapshotId; + } + + /** + * @param snapshotIdIn the snapshotId to set + */ + public void setSnapshotId(String snapshotIdIn) { + this.snapshotId = snapshotIdIn; + } + + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java new file mode 100644 index 0000000..1717109 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + + +/** + * BuilderFactory + * @version $Rev$ + */ +public class BuilderFactory { + + public static final String ERRATA_TYPE = "errata"; + public static final String PACKAGES_TYPE = "packages"; + public static final String SERVER_TYPE = "server"; + public static final String DOCS_TYPE = "docs"; + public static final String HARDWARE_DEVICE_TYPE = "hwdevice"; + public static final String SNAPSHOT_TAG_TYPE = "snapshotTag"; + public static final String SERVER_CUSTOM_INFO_TYPE = "serverCustomInfo"; + /** + * Private constructor. + */ + private BuilderFactory() { + } + + /** + * Returns a Builder suitable for building the wanted type. Invalid types + * will throw an UnsupportedOperationException. + * @param type Valid type of builder. + * @return Suitable builder. + */ + public static DocumentBuilder getBuilder(String type) { + if (ERRATA_TYPE.equals(type)) { + return new ErrataDocumentBuilder(); + } + else if (PACKAGES_TYPE.equals(type)) { + return new PackageDocumentBuilder(); + } + else if (SERVER_TYPE.equals(type)) { + return new ServerDocumentBuilder(); + } + else if (HARDWARE_DEVICE_TYPE.equals(type)) { + return new HardwareDeviceDocumentBuilder(); + } + else if (SNAPSHOT_TAG_TYPE.equals(type)) { + return new SnapshotTagDocumentBuilder(); + } + else if (SERVER_CUSTOM_INFO_TYPE.equals(type)) { + return new ServerCustomInfoDocumentBuilder(); + } + else { + throw new UnsupportedOperationException(type + + " is an unsuppported type"); + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java new file mode 100644 index 0000000..3cb733a --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + +import org.apache.lucene.document.Document; + +import java.util.Map; + + +/** + * DocumentBuilder + * @version $Rev$ + */ +public interface DocumentBuilder { + /** + * Builds a Lucene Document using the given object id and metadata map. + * The document will contain an "id" field with the value of the object id, + * and fields corresponding to the keys of the metadata and their values. + * @param objId Object id + * @param metadata key:value pairs describing the object. + * @return Lucene Document for indexing. + */ + Document buildDocument(Long objId, Map<String, String> metadata); +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java new file mode 100644 index 0000000..03fdd60 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import java.util.Iterator; +import java.util.Map; + + +/** + * ErrataDocumentBuilder + * @version $Rev$ + */ +public class ErrataDocumentBuilder implements DocumentBuilder { + + // TODO: add some information about fields and their indexing states. + + /** + * {@inheritDoc} + */ + public Document buildDocument(Long objId, Map<String, String> metadata) { + Document doc = new Document(); + doc.add(new Field("id", objId.toString(), Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + + for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { + Field.Store store = Field.Store.NO; + Field.Index tokenize = Field.Index.TOKENIZED; + + String name = iter.next(); + String value = metadata.get(name); + + if (name.equals("name")) { + store = Field.Store.YES; + } + else if (name.equals("advisoryName")) { + store = Field.Store.YES; + } + + doc.add(new Field(name, String.valueOf(value), store, + tokenize)); + } + return doc; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java new file mode 100644 index 0000000..b1fd2d2 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import java.util.Iterator; +import java.util.Map; + + +/** + * HardwareDeviceDocumentBuilder + * @version $Rev$ + */ +public class HardwareDeviceDocumentBuilder implements DocumentBuilder { + + /** + * {@inheritDoc} + */ + public Document buildDocument(Long objId, Map<String, String> metadata) { + Document doc = new Document(); + doc.add(new Field("id", objId.toString(), Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { + Field.Store store = Field.Store.YES; + Field.Index tokenize = Field.Index.TOKENIZED; + + String name = iter.next(); + String value = metadata.get(name); + + if (name.equals("name") || (name.equals("description"))) { + store = Field.Store.YES; + } + else if (name.equals("serverId")) { + store = Field.Store.YES; + tokenize = Field.Index.UN_TOKENIZED; + } + + doc.add(new Field(name, String.valueOf(value), store, + tokenize)); + } + return doc; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java new file mode 100644 index 0000000..624ad4f --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import java.util.Iterator; +import java.util.Map; + + +/** + * PackageDocumentBuilder + * @version $Rev$ + */ +public class PackageDocumentBuilder implements DocumentBuilder { + + /** + * {@inheritDoc} + */ + public Document buildDocument(Long objId, Map<String, String> metadata) { + Document doc = new Document(); + doc.add(new Field("id", objId.toString(), Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + for (Iterator<String> iter = metadata.keySet().iterator(); iter + .hasNext();) { + Field.Store store = Field.Store.YES; + Field.Index tokenize = Field.Index.TOKENIZED; + + String name = iter.next(); + String value = metadata.get(name); + if (name.equals("name")) { + store = Field.Store.YES; + } + else if (name.equals("arch") || name.equals("filename") || + name.equals("version") || name.equals("release") || + name.equals("epoch")) { + tokenize = Field.Index.UN_TOKENIZED; + } + + doc.add(new Field(name, String.valueOf(value), store, + tokenize)); + } + return doc; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java new file mode 100644 index 0000000..de10c7b --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import java.util.Iterator; +import java.util.Map; + + +/** + * ServerCustomInfoDocumentBuilder + * @version $Rev$ + */ +public class ServerCustomInfoDocumentBuilder implements DocumentBuilder { + + /** + * {@inheritDoc} + */ + public Document buildDocument(Long objId, Map<String, String> metadata) { + Document doc = new Document(); + doc.add(new Field("id", objId.toString(), Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { + Field.Store store = Field.Store.YES; + Field.Index tokenize = Field.Index.TOKENIZED; + + String name = iter.next(); + String value = metadata.get(name); + + if (name.equals("value")) { + store = Field.Store.YES; + } + else if (name.equals("created") || (name.equals("modified"))) { + store = Field.Store.YES; + tokenize = Field.Index.UN_TOKENIZED; + } + else if (name.equals("serverId") || name.equals("createdBy") || + name.equals("lastModifiedBy")) { + store = Field.Store.YES; + tokenize = Field.Index.UN_TOKENIZED; + } + + doc.add(new Field(name, String.valueOf(value), store, + tokenize)); + } + return doc; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java new file mode 100644 index 0000000..516c846 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import java.util.Iterator; +import java.util.Map; + + +/** + * ServerDocumentBuilder + * @version $Rev$ + */ +public class ServerDocumentBuilder implements DocumentBuilder { + + /** + * {@inheritDoc} + */ + public Document buildDocument(Long objId, Map<String, String> metadata) { + Document doc = new Document(); + // Keep 'id' UN_TOKENIZED, this is needed to determine 'uniqueness' + // if you tokenize this it will break deleting documents, we'll + // no longer have unique documents per id. + + doc.add(new Field("id", objId.toString(), Field.Store.YES, + Field.Index.UN_TOKENIZED)); + // This is the tokenized form of 'id' so we can do searches on it and + // use ngram flexibility + doc.add(new Field("system_id", objId.toString(), Field.Store.YES, + Field.Index.TOKENIZED)); + + for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { + Field.Store store = Field.Store.YES; + Field.Index tokenize = Field.Index.TOKENIZED; + + String name = iter.next(); + String value = metadata.get(name); + + if (name.equals("name") || name.equals("cpuModel") || + name.equals("hostname") || name.equals("ipaddr") || + (name.equals("runningKernel"))) { + store = Field.Store.YES; + } + else if (name.equals("checkin") || name.equals("registered") || + name.equals("ram") || name.equals("swap") || + name.equals("cpuMhz") | name.equals("cpuNumberOfCpus")) { + store = Field.Store.YES; + tokenize = Field.Index.UN_TOKENIZED; + } + + doc.add(new Field(name, String.valueOf(value), store, + tokenize)); + } + return doc; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java new file mode 100644 index 0000000..cf055c3 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import java.util.Iterator; +import java.util.Map; + + +/** + * SnapshotTagDocumentBuilder + * @version $Rev$ + */ +public class SnapshotTagDocumentBuilder implements DocumentBuilder { + + /** + * {@inheritDoc} + */ + public Document buildDocument(Long objId, Map<String, String> metadata) { + Document doc = new Document(); + doc.add(new Field("id", objId.toString(), Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { + Field.Store store = Field.Store.YES; + Field.Index tokenize = Field.Index.TOKENIZED; + + String name = iter.next(); + String value = metadata.get(name); + + if (name.equals("name")) { + store = Field.Store.YES; + } + else if (name.equals("created") || (name.equals("modified"))) { + store = Field.Store.YES; + tokenize = Field.Index.UN_TOKENIZED; + } + else if (name.equals("snapshotId") || name.equals("tagNameId") || + name.equals("serverId") || name.equals("orgId")) { + store = Field.Store.YES; + tokenize = Field.Index.UN_TOKENIZED; + } + + doc.add(new Field(name, String.valueOf(value), store, + tokenize)); + } + return doc; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java new file mode 100644 index 0000000..530e68c --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder.tests; + +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.ErrataDocumentBuilder; +import com.redhat.satellite.search.index.builder.PackageDocumentBuilder; +import com.redhat.satellite.search.index.builder.ServerDocumentBuilder; + +import junit.framework.TestCase; + + +/** + * BuilderFactoryTest + * @version $Rev$ + */ +public class BuilderFactoryTest extends TestCase { + + public void testGetBuilderErrataType() { + DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.ERRATA_TYPE); + assertNotNull(db); + assertEquals(db.getClass(), ErrataDocumentBuilder.class); + } + + public void testGetBuilderPackageType() { + DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.PACKAGES_TYPE); + assertNotNull(db); + assertEquals(db.getClass(), PackageDocumentBuilder.class); + } + + public void testGetBuilderServerType() { + DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.SERVER_TYPE); + assertNotNull(db); + assertEquals(db.getClass(), ServerDocumentBuilder.class); + } + + public void testGetBuilderInvalid() { + try { + BuilderFactory.getBuilder(null); + fail("getBuilder should have thrown exception with null type"); + } + catch(UnsupportedOperationException uoe) { + assertTrue(true); + } + + try { + BuilderFactory.getBuilder("foobar"); + fail("getBuilder should have thrown exception with null type"); + } + catch(UnsupportedOperationException uoe) { + assertTrue(true); + } + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java new file mode 100644 index 0000000..a4dcfc6 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder.tests; + +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.ErrataDocumentBuilder; + +import org.apache.lucene.document.Document; + +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + + +/** + * ErrataDocumentBuilderTest + * @version $Rev$ + */ +public class ErrataDocumentBuilderTest extends TestCase { + + public void testBuildDocument() { + Map<String,String> metadata = new HashMap<String,String>(); + metadata.put("id", new Long(10).toString()); + metadata.put("advisory", "Advisory"); + metadata.put("advisoryType", "AdvisoryType"); + metadata.put("advisoryName", "AdvisoryName"); + metadata.put("advisoryRel", new Long(100).toString()); + metadata.put("product", "Product"); + metadata.put("description", "Description"); + metadata.put("synopsis", "Synopsis"); + metadata.put("topic", "Topic"); + metadata.put("solution", "Solution"); + metadata.put("issueDate", "IssueDate"); + metadata.put("updateDate", "UpdateDate"); + metadata.put("notes", "Notes"); + metadata.put("orgId", "OrgId"); + metadata.put("created", "Created"); + metadata.put("modified", "Modified"); + metadata.put("lastModified", "LastModified"); + metadata.put("severityId", new Long(1).toString()); + metadata.put("name", "Advisory"); + + DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.ERRATA_TYPE); + assertTrue(db instanceof ErrataDocumentBuilder); + Document doc = db.buildDocument(new Long(10), metadata); + + assertNotNull(doc); + assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); + assertEquals(doc.getField("advisory").stringValue(), "Advisory"); + assertEquals(doc.getField("advisoryType").stringValue(), "AdvisoryType"); + assertEquals(doc.getField("advisoryName").stringValue(), "AdvisoryName"); + assertEquals(doc.getField("advisoryRel").stringValue(), new Long(100).toString()); + assertEquals(doc.getField("product").stringValue(), "Product"); + assertEquals(doc.getField("description").stringValue(), "Description"); + assertEquals(doc.getField("synopsis").stringValue(), "Synopsis"); + assertEquals(doc.getField("topic").stringValue(), "Topic"); + assertEquals(doc.getField("solution").stringValue(), "Solution"); + assertEquals(doc.getField("issueDate").stringValue(), "IssueDate"); + assertEquals(doc.getField("updateDate").stringValue(), "UpdateDate"); + assertEquals(doc.getField("notes").stringValue(), "Notes"); + assertEquals(doc.getField("orgId").stringValue(), "OrgId"); + assertEquals(doc.getField("created").stringValue(), "Created"); + assertEquals(doc.getField("modified").stringValue(), "Modified"); + assertEquals(doc.getField("lastModified").stringValue(), "LastModified"); + assertEquals(doc.getField("severityId").stringValue(), new Long(1).toString()); + assertEquals(doc.getField("name").stringValue(), "Advisory"); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java new file mode 100644 index 0000000..452a171 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder.tests; + +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.HardwareDeviceDocumentBuilder; + +import org.apache.lucene.document.Document; + +import junit.framework.TestCase; +import java.util.HashMap; +import java.util.Map; + + + +/** + * HardwareDeviceDocumentBuilderTest + * @version $Rev$ + */ +public class HardwareDeviceDocumentBuilderTest extends TestCase { + + public void testBuilderDocument() { + Map<String, String> metadata = new HashMap<String, String>(); + metadata.put("name", "Name"); + metadata.put("description", "Description"); + + DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.HARDWARE_DEVICE_TYPE); + assertTrue(db instanceof HardwareDeviceDocumentBuilder); + Document doc = db.buildDocument(new Long(10), metadata); + + assertNotNull(doc); + assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); + assertEquals(doc.getField("name").stringValue(), "Name"); + assertEquals(doc.getField("description").stringValue(), "Description"); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java new file mode 100644 index 0000000..2d77640 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder.tests; + +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.PackageDocumentBuilder; + +import org.apache.lucene.document.Document; + +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + + +/** + * PackageDocumentBuilderTest + * @version $Rev$ + */ +public class PackageDocumentBuilderTest extends TestCase { + + public void testBuildDocument() { + Map<String, String> metadata = new HashMap<String, String>(); + metadata.put("name", "Name"); + metadata.put("version", "PrettyVersion"); + metadata.put("filename", "FileName"); + metadata.put("description", "Description"); + metadata.put("summary", "Summary"); + metadata.put("arch", "Arch"); + + DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.PACKAGES_TYPE); + assertTrue(db instanceof PackageDocumentBuilder); + Document doc = db.buildDocument(new Long(10), metadata); + + assertNotNull(doc); + assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); + assertEquals(doc.getField("name").stringValue(), "Name"); + assertEquals(doc.getField("version").stringValue(), "PrettyVersion"); + assertEquals(doc.getField("filename").stringValue(), "FileName"); + assertEquals(doc.getField("description").stringValue(), "Description"); + assertEquals(doc.getField("summary").stringValue(), "Summary"); + assertEquals(doc.getField("arch").stringValue(), "Arch"); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java new file mode 100644 index 0000000..89935a3 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder.tests; + +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.ServerCustomInfoDocumentBuilder; + +import org.apache.lucene.document.Document; + +import junit.framework.TestCase; +import java.util.HashMap; +import java.util.Map; + + + +/** + * ServerCustomInfoDocumentBuilderTest + * @version $Rev$ + */ +public class ServerCustomInfoDocumentBuilderTest extends TestCase { + + public void testBuilderDocument() { + Map<String, String> metadata = new HashMap<String, String>(); + metadata.put("value", "value"); + metadata.put("serverId", "server id"); + + DocumentBuilder db = BuilderFactory.getBuilder( + BuilderFactory.SERVER_CUSTOM_INFO_TYPE); + assertTrue(db instanceof ServerCustomInfoDocumentBuilder); + Document doc = db.buildDocument(new Long(10), metadata); + + assertNotNull(doc); + assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); + assertEquals(doc.getField("value").stringValue(), "value"); + assertEquals(doc.getField("serverId").stringValue(), "server id"); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java new file mode 100644 index 0000000..c1ee3e3 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder.tests; + +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.ServerDocumentBuilder; + +import org.apache.lucene.document.Document; + +import junit.framework.TestCase; +import java.util.HashMap; +import java.util.Map; + + + +/** + * ServerDocumentBuilderTest + * @version $Rev$ + */ +public class ServerDocumentBuilderTest extends TestCase { + + public void testBuilderDocument() { + Map<String, String> metadata = new HashMap<String, String>(); + metadata.put("name", "Name"); + metadata.put("description", "Description"); + metadata.put("info", "Info"); + + DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.SERVER_TYPE); + assertTrue(db instanceof ServerDocumentBuilder); + Document doc = db.buildDocument(new Long(10), metadata); + + assertNotNull(doc); + assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); + assertEquals(doc.getField("name").stringValue(), "Name"); + assertEquals(doc.getField("description").stringValue(), "Description"); + assertEquals(doc.getField("info").stringValue(), "Info"); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java new file mode 100644 index 0000000..6b0fe32 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.builder.tests; + +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.SnapshotTagDocumentBuilder; + +import org.apache.lucene.document.Document; + +import junit.framework.TestCase; +import java.util.HashMap; +import java.util.Map; + + + +/** + * SnapshotTagDocumentBuilderTest + * @version $Rev$ + */ +public class SnapshotTagDocumentBuilderTest extends TestCase { + + public void testBuilderDocument() { + Map<String, String> metadata = new HashMap<String, String>(); + metadata.put("name", "Name"); + + DocumentBuilder db = + BuilderFactory.getBuilder(BuilderFactory.SNAPSHOT_TAG_TYPE); + assertTrue(db instanceof SnapshotTagDocumentBuilder); + Document doc = db.buildDocument(new Long(10), metadata); + + assertNotNull(doc); + assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); + assertEquals(doc.getField("name").stringValue(), "Name"); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java new file mode 100644 index 0000000..fb39ca0 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.index.ngram; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.analysis.standard.StandardFilter; +import org.apache.lucene.analysis.TokenStream; +// From Lucene Sandbox +import org.apache.lucene.analysis.ngram.NGramTokenFilter; + +import java.io.Reader; + +/** + * NGramAnalyzer + * A ngram will take a term and break it up into a series of smaller + * permutations of different letter combinations. + * @version $Rev$ + */ +public class NGramAnalyzer extends Analyzer { + //Controls minimum size ngram to construct + protected int min_ngram; + //Controls maximum size ngram to construct + protected int max_ngram; + + /** + * Constructor + * @param min min length of ngram to generate + * @param max max length of ngram to generate + */ + public NGramAnalyzer(int min, int max) { + super(); + min_ngram = min; + max_ngram = max; + } + /** + * Constructs a pre populated + * @param reader contains data to parse + * @return TokenStream of ngrams + */ + public TokenStream tokenStream(Reader reader) { + return tokenStream(null, reader); + } + + /** + * @param fieldName ignored param + * @param reader contains data to parse + * @return TokenStream of ngrams + */ + public TokenStream tokenStream(String fieldName, Reader reader) { + return new NGramTokenFilter( + new LowerCaseFilter( + new StandardFilter( + new StandardTokenizer(reader))), min_ngram, max_ngram); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java new file mode 100644 index 0000000..8d14250 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + + +package com.redhat.satellite.search.index.ngram; + +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.index.Term; + +import java.io.IOException; +import java.io.StringReader; + +/** + * NGramQuery + * A custom BooleanQuery, it takes each ngram-token and adds as an OR term. + * @version $Rev$ + */ +public class NGramQuery extends BooleanQuery { + + private static final long serialVersionUID = 1L; + + /** + * Constructor + * @param field name of the field + * @param queryTerms String containing a term or a series of terms to search. + * The string will be parsed and will be broken up into a series of NGrams. + * @throws IOException something went wrong parsing queryTerms + * */ + public NGramQuery(String field, String queryTerms, int min, int max) + throws IOException { + NGramAnalyzer nga = new NGramAnalyzer(min, max); + TokenStream ngrams = nga.tokenStream(new StringReader(queryTerms)); + Token token; + while ((token = ngrams.next()) != null) { + Term t = new Term(field, new String(token.termBuffer()).trim()); + add(new TermQuery(t), BooleanClause.Occur.SHOULD); + } + } + + /** + * + * @param pq PhraseQuery to break up and convert to NGramQuery + * Forms a BooleanQuery with each term in the original PhraseQuery OR'd. + * Note: Assumes that each term has already been tokenized into a ngram, + * this method will not re-tokenize terms. + * @param useMust controls if BooleanClause.Occur SHOULD or MUST is used. + */ + public NGramQuery(PhraseQuery pq, boolean useMust) { + Term[] terms = pq.getTerms(); + for (int i = 0; i < terms.length; i++) { + BooleanClause.Occur occur = BooleanClause.Occur.SHOULD; + if (useMust) { + occur = BooleanClause.Occur.MUST; + } + add(new TermQuery(terms[i]), occur); + } + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java new file mode 100644 index 0000000..7ae2311 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java @@ -0,0 +1,149 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.ngram; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.lucene.search.ConstantScoreRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.DateTools; +import org.apache.lucene.document.NumberTools; + +import org.apache.log4j.Logger; + +/** + * NGramQueryParser + * Creates a custom query parser of ngram-tokenized search terms + * + * @version $Rev$ + */ +public class NGramQueryParser extends QueryParser { + + private static Logger log = Logger.getLogger(NGramQueryParser.class); + private boolean useMust = false; + /** + * Constructor + * @param f field name + * @param a analyzer + * @param useMustIn boolean option set to true when doing a Free Form or + * advanced search, it will force each NGramQuery to be constructed using + * the BooleanClause.Occur.MUST. This narrows returned search results and + * is not recommended for regular searches which want spelling variances + * to be tolerated. + */ + public NGramQueryParser(String f, Analyzer a, boolean useMustIn) { + super(f, a); + this.setDateResolution(DateTools.Resolution.DAY); + this.useMust = useMustIn; + } + + /** + * Constructor + * @param f field name + * @param a analyzer + */ + public NGramQueryParser(String f, Analyzer a) { + super(f, a); + this.setDateResolution(DateTools.Resolution.DAY); + } + + /** + * + * @return value for useMust + */ + public boolean getUseMust() { + return useMust; + } + + /** + * + * @param value for useMust + */ + public void setUseMust(boolean value) { + useMust = value; + } + + protected Query getFieldQuery(String defaultField, + String queryText) throws ParseException { + Query orig = super.getFieldQuery(defaultField, queryText); + if (!(orig instanceof PhraseQuery)) { + log.debug("Returning default query. No phrase query translation."); + return orig; + } + /** + * A ngram when parsed will become a series of smaller search terms, + * these terms are grouped together into a PhraseQuery. We are taking + * that PhraseQuery and breaking out each ngram term then combining all + * ngrams together to form a BooleanQuery. + */ + PhraseQuery pq = (PhraseQuery)orig; + return new NGramQuery(pq, useMust); + } + + /** + * + * @param field + * @return return true if this looks to be a date string + * false if this is not a date string + */ + protected boolean isDate(String field) { + if (field.length() == 12) { + return true; + } + return false; + } + /** + * This will look to see if "part1" or "part2" are strings of all digits, + * if they are, then they will be converted to a lexicographically safe string + * representation, then passed into the inherited getRangeQuery(). This is needed when + * comparing something like "4" to be less than "10". + * If the strings don't fit the pattern of all digits, then they get passed through + * to the inherited getRangeQuery(). + */ + protected Query getRangeQuery(String field, + String part1, + String part2, + boolean inclusive) throws ParseException { + if (isDate(part1) && isDate(part2)) { + if (log.isDebugEnabled()) { + log.debug("Detected passed in terms are dates, creating " + + "ConstantScoreRangeQuery(" + field + ", " + part1 + ", " + + part2 + ", " + inclusive + ", " + inclusive); + } + return new ConstantScoreRangeQuery(field, part1, part2, inclusive, + inclusive); + } + String newPart1 = part1; + String newPart2 = part2; + String regEx = "(\d)*"; + Pattern pattern = Pattern.compile(regEx); + Matcher matcher1 = pattern.matcher(part1); + Matcher matcher2 = pattern.matcher(part2); + if (matcher1.matches() && matcher2.matches()) { + newPart1 = NumberTools.longToString(Long.parseLong(part1)); + newPart2 = NumberTools.longToString(Long.parseLong(part2)); + if (log.isDebugEnabled()) { + log.debug("NGramQueryParser.getRangeQuery() Converted " + part1 + " to " + + newPart1 + ", Converted " + part2 + " to " + newPart2); + } + } + return super.getRangeQuery(field, newPart1, newPart2, inclusive); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java new file mode 100644 index 0000000..0ed65f3 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.ngram.tests; + +import com.redhat.satellite.search.index.ngram.NGramAnalyzer; + +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; + +import org.apache.log4j.Logger; + +import java.io.StringReader; + +public class NGramAnalyzerTest extends NGramTestSetup { + + private static Logger log = Logger.getLogger(NGramAnalyzerTest.class); + + public NGramAnalyzerTest() { + super(); + } + + public void testTokenStream() throws Exception { + NGramAnalyzer nga = new NGramAnalyzer(min_ngram, max_ngram); + TokenStream ngrams = nga.tokenStream(new StringReader("aspell")); + Token token; + String result = new String(""); + while ((token = ngrams.next()) != null) { + result += new String(token.termBuffer()).trim() + ","; + } + log.info("Created a ngram token stream, this is what it looks like: " + + result); + + assertTrue("testTokenStream", result.compareTo("a,s,p,e,l,l,as,sp,pe," + + "el,ll,asp,spe,pel,ell,aspe,spel,pell,aspel,spell,") == 0); + + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java new file mode 100644 index 0000000..e5077bb --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java @@ -0,0 +1,210 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.ngram.tests; + +import java.io.IOException; + +import com.redhat.satellite.search.index.ngram.NGramAnalyzer; +import com.redhat.satellite.search.index.ngram.NGramQueryParser; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; + +public class NGramQueryParserTest extends NGramTestSetup { + + private static Logger log = Logger.getLogger(NGramQueryParserTest.class); + + public NGramQueryParserTest() { + super(); + } + + public void testBasicQueryParse() throws Exception { + String defaultField = new String("name"); + NGramQueryParser parser = new NGramQueryParser(defaultField, new NGramAnalyzer(min_ngram, max_ngram)); + String txt = new String("spell"); + Query q = parser.parse(txt); + log.info("testBasicQueryParse() query = " + q.toString()); + assertTrue(q.toString().compareTo("name:s name:p name:e name:l name:l " + + "name:sp name:pe name:el name:ll name:spe name:pel name:ell " + + "name:spel name:pell name:spell") == 0); + } + + /** + * We want to make sure that when searching for multiple terms, each term will become it's + * own BooleanQuery. + * */ + public void testMultiPhraseQueryParse() throws Exception { + String defaultField = new String("name"); + NGramQueryParser parser = new NGramQueryParser(defaultField, new NGramAnalyzer(min_ngram, max_ngram)); + String txt = new String("spell* virt manager"); + Query q = parser.parse(txt); + assertTrue(q instanceof BooleanQuery); + BooleanQuery bq = (BooleanQuery)q; + assertTrue(bq.getClauses().length == 3); + } + + public void testWildcardQueryParse() throws Exception { + NGramQueryParser parser = new NGramQueryParser("name", new NGramAnalyzer(min_ngram, max_ngram)); + String txt = new String("spell*"); + Query q = parser.parse(txt); + log.info("Wildcard query = " + q.toString()); + assertTrue(q.toString().compareTo("name:spell*") == 0); + } + + public void testQueryParseWithSpecialChars() throws Exception { + String queryString = new String("spell* virt- manager+"); + log.info("testQueryParserWithSpecialChars(): query string is: " + queryString); + NGramQueryParser parser = new NGramQueryParser("name", new NGramAnalyzer(min_ngram, max_ngram)); + Query q = parser.parse(queryString); + log.info("Using NGramQueryParser query = " + q.toString()); + + QueryParser origParser = new QueryParser("name", new StandardAnalyzer()); + q = origParser.parse(queryString); + log.info("Using QueryParser query = " + q.toString()); + } + + public Hits performSearch(Directory dir, String query, boolean useMust) + throws Exception { + + NGramQueryParser parser = new NGramQueryParser("name", + new NGramAnalyzer(min_ngram, max_ngram), useMust); + IndexSearcher searcher = new IndexSearcher(dir); + Query q = parser.parse(query); + Hits hits = searcher.search(q); + log.info("Original Query = " + query); + log.info("Parsed Query = " + q.toString()); + log.info("Hits.length() = " + hits.length()); + for (int i=0; i < hits.length(); i++) { + log.debug("Document<"+hits.id(i)+"> = " + hits.doc(i)); + //Explanation explain = searcher.explain(q, hits.id(i)); + //log.debug("explain = " + explain.toString()); + } + return hits; + } + + public void testFreeFormQueryParse() throws Exception { + String queryString = new String("name:spell -description:another"); + log.info("Original query: " + queryString); + + NGramQueryParser parser = new NGramQueryParser("name", + new NGramAnalyzer(min_ngram, max_ngram), true); + Query q = parser.parse(queryString); + log.info("NGramQueryParser parsed query: " + q.toString()); + + QueryParser origParser = new QueryParser("name", new StandardAnalyzer()); + q = origParser.parse(queryString); + log.info("QueryParser parsed query = " + q.toString()); + } + + + + public void testFreeFormSearch() throws Exception { + Hits hits = null; + String query = null; + boolean useMust = true; + // Grab all packages with name "spell" AND + // description does NOT contain "another" + query = "name:spell -description:another"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(hits.length() == 2); + + // Grab all packages with name "virt" AND + // description MUST have "factory" in it + query = "name:virt +description:factory"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(hits.length() == 2); + + // Grab all packages with name "virt" + query = "name:virt description:factory"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(hits.length() == 4); + + query = "name:virt OR description:factory"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(hits.length() == 4); + + query = "name:virt AND description:factory"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(hits.length() == 1); + + query = "name:virt -description:factory"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(hits.length() == 2); + } + /** + * + * */ + public void testBasicSearch() throws Exception { + Hits hits; + String query; + boolean useMust = false; + query = "spell"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(thresholdHits(hits) == 5); + assertTrue(hits.length() == 16); + + query = "aspelll"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(thresholdHits(hits) == 4); + assertTrue(hits.length() == 17); + + query = "aspell"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(thresholdHits(hits) == 4); + assertTrue(hits.length() == 17); + + query = "pel"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(thresholdHits(hits) == 8); + assertTrue(hits.length() == 16); + + query = "gtk"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(thresholdHits(hits) == 7); + assertTrue(hits.length() == 17); + + + // We want a search for kernel-hugemem to return kernel-hugemem as top hit + // but currently, kernel-hugemem-devel is matchin instead. This test + // is a placeholder as we explore ways to fix this. + query = "((name:kernel-hugemem)^2 (description:kernel-hugemem) " + + "(filename:kernel-hugemem))"; + hits = performSearch(this.ngramDir, query, useMust); + displayHits(hits); + assertTrue(thresholdHits(hits) == 3); + assertTrue(hits.length() == 20); + String firstHitName = hits.doc(0).get("name"); + assertTrue(firstHitName.compareToIgnoreCase("kernel-hugemem-devel") == 0); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java new file mode 100644 index 0000000..d8dadb2 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.ngram.tests; + +import com.redhat.satellite.search.index.ngram.NGramQuery; + +import org.apache.lucene.search.Query; + +import org.apache.log4j.Logger; + +public class NGramQueryTest extends NGramTestSetup { + + private static Logger log = Logger.getLogger(NGramQueryTest.class); + + public NGramQueryTest() { + super(); + } + + public void testCreateQuery() throws Exception { + String term = "spell"; + Query q = new NGramQuery("name", term, min_ngram, max_ngram); + log.info("NGramQuery("+term+") = " + q.toString()); + assertTrue(q != null); + } + + public void testCreateQueryMinMax() throws Exception { + String term = "spell"; + Query q = new NGramQuery("name", term, min_ngram, max_ngram); + log.info("NGramQuery("+term+") = " + q.toString()); + assertTrue(q.toString().compareTo("name:s name:p name:e name:l " + + "name:l name:sp name:pe name:el name:ll name:spe name:pel " + + "name:ell name:spel name:pell name:spell") == 0); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java new file mode 100644 index 0000000..d3ac974 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java @@ -0,0 +1,163 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.index.ngram.tests; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + + +import com.redhat.satellite.search.index.ngram.NGramAnalyzer; + +import org.apache.log4j.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; + +import junit.framework.TestCase; + + +public class NGramTestSetup extends TestCase { + private static Logger log = Logger.getLogger(NGramTestSetup.class); + + protected RAMDirectory ngramDir; + protected RAMDirectory stanDir; + + protected double score_threshold = .10; + protected int min_ngram = 1; + protected int max_ngram = 5; + + protected List<Map<String,String>> items = + new LinkedList<Map<String, String>>(); + + public NGramTestSetup() { + super(); + } + + protected void addItem(String name, String description, String filename) { + Map<String, String> item = new HashMap<String, String>(); + item.put("name", name); + item.put("description", description); + item.put("filename", filename); + items.add(item); + } + + protected void initItems() { + addItem("spellingbee", "spelling application", "spellingbee-1.0.rpm"); + addItem("aspelling", "another spelling program alternate spell", + "aspelling-1.0.rpm"); + addItem("aspell", "another spelling program", "aspell-0.3.4.rpm"); + addItem("spell", "spelling program", "spell-4943.rpm"); + addItem("slelp", "application with spelling error", "slelp1-43.rpm"); + addItem("libvirt", "virtualization library", "virt-pkg-1.rpm"); + addItem("virt-manager", "blah blah application", "virt-pkg-1.rpm"); + addItem("virtualfactory", "virtual factory something", + "virtfact-04.rpm"); + addItem("newFactory", "factory test application", "newFactory-1.9.rpm"); + addItem("gtk+-devel", "development library for gtk", + "gtk+-devel-10.rpm"); + addItem("gtk+", "runtime library", "gtk+-30.rpm"); + addItem("gtk-doc", "documentation for gtk", "gtk-doc-393.rpm"); + addItem("authconfig-gtk", "authentication related gtk", + "authconfig-gtk-039.rpm"); + addItem("mtr-gtk", "blah blah mtr gtk", "mtr-gtk-039.rpm"); + addItem("ghostscript-gtk", "printting support application gtk", + "ghostscript-gtk-30.rpm"); + addItem("gnome-bluetooth-libs", "library for bluetooth support", + "gnome-bluetooth-libs-3.4.rpm"); + addItem("scim-bridge-gtk", "blah blah scim gtk", + "scim-bridge-gtk-494.rpm"); + addItem("kernel", "linux kernel package", "kernel-094.rpm"); + addItem("kernel-hugemem", "This package includes an SMP version of " + + "the Linux kernel which supports systems with 16 Gigabytes " + + "of memory or more.", "kernel-hugemem-2.6.9-84.EL.i686"); + addItem("kernel-hugemem-devel", "This package provides kernel " + + "headers +and makefiles sufficient to build modules against " + + "the hugemem kernel package.", + "kernel-hugemem-devel-2.6.9-84.EL.i686"); + } + /** + * Creates an index in RAM + * */ + public void setUp() throws Exception { + super.setUp(); + initItems(); + this.stanDir = new RAMDirectory(); + IndexWriter stanWriter = new IndexWriter(this.stanDir, new StandardAnalyzer(), true); + + this.ngramDir = new RAMDirectory(); + IndexWriter ngramWriter = new IndexWriter(this.ngramDir, new NGramAnalyzer(min_ngram, max_ngram), true); + + for (Map<String, String> item: items) { + String name = item.get("name"); + String descp = item.get("description"); + Document doc = new Document(); + doc.add(new Field("name", name, Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("description", descp, Field.Store.YES, + Field.Index.TOKENIZED)); + stanWriter.addDocument(doc); + ngramWriter.addDocument(doc); + } + stanWriter.close(); + ngramWriter.close(); + } + + public Hits performSearch(Directory dir, Analyzer alyz, String query) throws Exception { + QueryParser parser = new QueryParser("name", alyz); + IndexSearcher searcher = new IndexSearcher(dir); + Query q = parser.parse(query); + Hits hits = searcher.search(q); + return hits; + } + + protected int thresholdHits(Hits hits) throws IOException { + /** We could consider doing thresholding as a relative thing... + * instead of checking against an absolute value, we grab top score + * then filter based on difference from that... + */ + int counter = 0; + for (int i=0; i < hits.length(); i++) { + if (hits.score(i) >= score_threshold) { + counter++; + } + else { + break; + } + } + return counter; + } + + protected void displayHits(Hits hits) throws IOException { + for (int i = 0; i < hits.length(); i++) { + Document doc = hits.doc(i); + String name = doc.get("name"); + String description = doc.get("description"); + log.info("Hit<" + i + "> Score< " + hits.score(i) + "> name = <" + + name + "> description = <" + description + ">"); + } + } + + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java new file mode 100644 index 0000000..238073f --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java @@ -0,0 +1,93 @@ +package com.redhat.satellite.search.index.tests; + +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.index.Result; +import com.redhat.satellite.search.index.QueryParseException; +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; +import com.redhat.satellite.search.index.builder.PackageDocumentBuilder; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; +import com.redhat.satellite.search.config.Configuration; + +import org.apache.lucene.document.Document; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IndexManagerTest extends BaseTestCase { + + private IndexManager indexManager; + + public void setUp() throws Exception { + super.setUp(); + indexManager = (IndexManager) + container.getComponentInstance(IndexManager.class); + } + + public void testIndexing() throws IOException, IndexingException { + String index = "foo"; + Long objectId = new Long(123); + Map<String, String> meta = new HashMap<String, String>(); + meta.put("name", "foo"); + meta.put("desc", "A really nice foo"); + meta.put("size", "12345"); + meta.put("dateCreated", "7/13/2007"); + DocumentBuilder pdb = new PackageDocumentBuilder(); + Document doc = pdb.buildDocument(objectId, meta); + indexManager.addToIndex(index, doc, "en"); + } + + public void testQuerying() + throws IOException, IndexingException, QueryParseException { + + String index = "foo"; + Long objectId = new Long(123); + Map<String, String> meta = new HashMap<String, String>(); + meta.put("name", "foo"); + meta.put("desc", "A really nice foo"); + meta.put("size", "12345"); + meta.put("dateCreated", "7/13/2007"); + DocumentBuilder pdb = new PackageDocumentBuilder(); + Document doc = pdb.buildDocument(objectId, meta); + indexManager.addToIndex(index, doc, "en"); + List<Result> results = indexManager.search(index, "name:foo", "en"); + assertTrue(results.size() >= 1); + results = indexManager.search(index, "desc:really", "en"); + assertTrue(results.size() >= 1); + } + + + public void StillNeedsWork_testQueryDocs() + throws IOException, IndexingException, QueryParseException { + + Configuration config = TestUtil.makeConfiguration(); + config.setString("search.index_work_dir", "/usr/share/rhn/search/indexes/"); + IndexManager indexMgr = new IndexManager(config); + // + // TODO: Revisit how docs data gets injected for testing. + // Currently relying on docs data to already exist for now. + // + System.out.println("We are expecting nutch to have been run previously, " + + "and the index files to be copied to "+ config.getString("search.index_work_dir") + + BuilderFactory.DOCS_TYPE); + + String index = BuilderFactory.DOCS_TYPE; + String query = new String("redhat"); + List<Result> results = indexMgr.search(index, query, "en"); + System.out.println("Number of results returned is " + results.size()); + assertTrue(results.size() >= 1); + + } + + @SuppressWarnings("unchecked") + @Override + protected Class[] getComponentClasses() { + return TestUtil.buildComponentsList(IndexManager.class); + } + + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java new file mode 100644 index 0000000..fa1c516 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java @@ -0,0 +1,26 @@ +package com.redhat.satellite.search.index.tests; + +import com.redhat.satellite.search.index.KeywordAnalyzer; + +import org.apache.lucene.analysis.TokenStream; + +import junit.framework.TestCase; + +import java.io.StringReader; + +public class KeywordAnalyzerTest extends TestCase { + + public void processString(String originalValue) throws Exception { + KeywordAnalyzer ka = new KeywordAnalyzer(); + StringReader sr = new StringReader(originalValue); + TokenStream ts = ka.tokenStream("ignoredField", sr); + assertTrue("Text Should be Untouched", new String(ts.next().termBuffer()).trim(). + compareTo(originalValue) == 0); + assertTrue("Token should be null", ts.next() == null); + } + public void testBasicParse() throws Exception { + processString("i386"); + processString("bx-gh-3&^0-993$#@!%^&*()-=+_><?/.,';:[]}{)"); + processString("j839,. 43 ..,.-=-=`1~!@#"); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/RpcServer.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/RpcServer.java new file mode 100644 index 0000000..695c301 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/RpcServer.java @@ -0,0 +1,159 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc; + +import com.redhat.satellite.search.config.Configuration; +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.scheduler.ScheduleManager; + +import org.picocontainer.Startable; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.util.Iterator; +import java.util.Map; + +import redstone.xmlrpc.XmlRpcServer; +import simple.http.connect.Connection; +import simple.http.connect.ConnectionFactory; + +/** + * Manages the embedded webserver and configures the XML-RPC runtime + * + * @version $Rev$ + */ +public class RpcServer implements Startable { + + private int listenPort; + private InetAddress listenAddress; + private Map<String, String> handlers; + private XmlRpcServer xmlrpcServer; + private ServerSocket socket; + private IndexManager indexManager; + private DatabaseManager databaseManager; + private ScheduleManager scheduleManager; + + /** + * Constructor + * + * @param config + * dependency + * @param idxManager + * dependency + * @throws UnknownHostException + * if config contains bad rpc_address + */ + public RpcServer(Configuration config, IndexManager idxManager, + DatabaseManager dbManager, ScheduleManager schedMgr) throws + UnknownHostException { + listenPort = config.getInt("search.rpc_port", 2828); + String addr = config.getString("search.rpc_address", "127.0.0.1"); + if (addr != null) { + listenAddress = InetAddress.getByName(addr); + } + handlers = config.getMap("search.rpc_handlers"); + if (handlers == null) { + throw new IllegalArgumentException( + "search.rpc_handlers config entry " + "is missing"); + } + indexManager = idxManager; + databaseManager = dbManager; + scheduleManager = schedMgr; + } + + /** + * Starts the server + * @throws RuntimeException something bad happened + */ + public void start() { + xmlrpcServer = new XmlRpcServer(); + for (Iterator<String> iter = handlers.keySet().iterator(); iter + .hasNext();) { + String name = iter.next(); + String className = handlers.get(name); + xmlrpcServer.addInvocationHandler(name, loadHandler(className)); + } + XmlRpcInvoker invoker = new XmlRpcInvoker(xmlrpcServer); + Connection connection = ConnectionFactory.getConnection(invoker); + try { + socket = new ServerSocket(); + SocketAddress sockAddr = null; + if (listenAddress != null) { + sockAddr = new InetSocketAddress(listenAddress, listenPort); + } + else { + sockAddr = new InetSocketAddress(listenPort); + } + socket.bind(sockAddr); + connection.connect(socket); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + + /** + * Stops the server + * + * @throws IOException + */ + public void stop() { + try { + socket.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + @SuppressWarnings("unchecked") + private Object loadHandler(String className) { + try { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Class klass = cl.loadClass(className); + Class[] paramTypes = { IndexManager.class, DatabaseManager.class, + ScheduleManager.class }; + Object[] params = { indexManager, databaseManager, scheduleManager}; + Constructor c = klass.getConstructor(paramTypes); + return c.newInstance(params); + } + catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + catch (IllegalArgumentException e) { + throw new RuntimeException(e); + } + catch (InstantiationException e) { + throw new RuntimeException(e); + } + catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + catch (InvocationTargetException e) { + throw new RuntimeException(e); + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java new file mode 100644 index 0000000..0352d05 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java @@ -0,0 +1,96 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc; + +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.StringWriter; + +import redstone.xmlrpc.XmlRpcServer; +import simple.http.ProtocolHandler; +import simple.http.Request; +import simple.http.Response; + +/** + * SimpleWeb ProtocolHandler used to pass requests to the XML-RPC layer + * + * @version $Rev$ + * + */ +public class XmlRpcInvoker implements ProtocolHandler { + + private static Logger log = Logger.getLogger(XmlRpcInvoker.class); + private XmlRpcServer server; + + /** + * Constructor + * + * @param xmlrpcServer + * handles actual XML-RPC calls + */ + public XmlRpcInvoker(XmlRpcServer xmlrpcServer) { + server = xmlrpcServer; + } + + /** + * {@inheritDoc} + */ + public void handle(Request request, Response response) { + String uri = request.getURI(); + log.info(uri); + try { + if (!uri.startsWith("/RPC2")) { + response.setCode(404); + response.setText(uri); + PrintStream out = response.getPrintStream(); + out.println("<html><body><title>Page not found</title>"); + out.println("<b>" + uri + " not found</b>"); + out.println("</body></html>"); + response.set("Content-Type", "text/html"); + out.flush(); + out.close(); + } + else { + InputStream in = request.getInputStream(); + StringWriter writer = new StringWriter(); + server.execute(in, writer); + OutputStream out = response.getOutputStream(); + response.set("Content-Type", "text/xml"); + writer.flush(); + out.write(writer.getBuffer().toString().getBytes()); + out.flush(); + out.close(); + } + } + catch (IOException e) { + e.printStackTrace(); + } + finally { + try { + response.commit(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java new file mode 100644 index 0000000..3222978 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc.handlers; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.scheduler.ScheduleManager; + +import org.apache.log4j.Logger; + +/** + * XML-RPC handler which handles calls for administration + * Updating indexes maybe more tasks later + * + * @version $Rev$ + */ +public class AdminHandler { + + private static Logger log = Logger.getLogger(AdminHandler.class); + private ScheduleManager scheduleManager; + + /** + * Constructor + * + * @param idxManager + * Search engine interface + */ + public AdminHandler(IndexManager idxManager, DatabaseManager dbMgr, + ScheduleManager schedMgr) { + log.info("** AdminHandler constructor invoked"); + scheduleManager = schedMgr; + } + /** + * Causes the task associated with the indexName to run and index new data. + * + * @param indexName + * @return true if index update is scheduled, false if unable to schedule. + */ + public boolean updateIndex(String indexName) { + if (log.isDebugEnabled()) { + log.debug("AdminHandler::updateIndex(" + indexName); + } + return scheduleManager.triggerIndexTask(indexName); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java new file mode 100644 index 0000000..85e9872 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java @@ -0,0 +1,177 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc.handlers; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.Query; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.Result; +import com.redhat.satellite.search.index.QueryParseException; +import com.redhat.satellite.search.scheduler.ScheduleManager; + +import com.ibatis.sqlmap.client.SqlMapException; + +import org.apache.log4j.Logger; + +import redstone.xmlrpc.XmlRpcFault; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * XML-RPC handler which handles calls for database queries + * + * TODO: + * 1) Restrict this class to only being able to call queries + * defined in specified namespaces, for now that would be: "errataHandler", + * errata_handler.xml. + * 2) Screenhits returned, verify these objects are allowed to be seen by the logged in + * user. + * + * + * @version $Rev$ + */ +public class DatabaseHandler { + + private static Logger log = Logger.getLogger(DatabaseHandler.class); + private DatabaseManager databaseManager; + /** + * Constructor + * @param idxManager IndexManager - unused param, needed for interface compatibility + * with RpcServer constructing us. + * @param dbMgr DatabaseManager + * Will perform database queries + * @param schedManager ScheduleManager unused param, needed for interface compatibility + * with RpcServer constructing us. + */ + public DatabaseHandler(IndexManager idxManager, DatabaseManager dbMgr, + ScheduleManager schedManager) { + databaseManager = dbMgr; + } + + /** + * Search database + * + * @param sessionId + * user's application session id + * @param namespace + * namespace query is located in + * @param query + * search query + * @return list of document ids as results + * @throws XmlRpcFault something bad happened + */ + public List<Result> search(long sessionId, String namespace, String query) + throws XmlRpcFault { + if (log.isDebugEnabled()) { + log.debug("DatabaseHandler:: searching for <" + namespace + ">: " + query); + } + try { + String queryName = getQueryName(query); + Map<String, String> params = getQueryParams(query); + if (log.isDebugEnabled()) { + log.debug("Calling runQuery(" + sessionId + ", " + namespace + + ", " + queryName + ", " + params + ")"); + } + List<Result> hits = runQuery(sessionId, namespace, queryName, + params); + log.info("Returned " + hits.size() + " records"); + return hits; + } + catch (SqlMapException e) { + log.warn("Caught SqlMapException: " + e.getMessage()); + e.printStackTrace(); + throw new XmlRpcFault(IndexHandler.DB_ERROR, e.getMessage()); + } + catch (SQLException e) { + log.warn("Caught SQLException: " + e.getMessage()); + e.printStackTrace(); + throw new XmlRpcFault(IndexHandler.DB_ERROR, e.getMessage()); + } + catch (QueryParseException e) { + log.warn("Caught QueryParseException: " + e.getMessage()); + e.printStackTrace(); + throw new XmlRpcFault(IndexHandler.QUERY_ERROR, e.getMessage()); + } + } + + /** + * + * @param queryIn format "name(param1, param2, ....)" + * @return query name from query + * @throws QueryParseException thrown when queryIn is malformed + */ + private String getQueryName(String queryIn) throws QueryParseException { + queryIn = queryIn.trim(); + int index = queryIn.indexOf(":("); + if (index < 0) { + throw new QueryParseException("Could not parse query: '" + queryIn + "'"); + } + String queryName = queryIn.substring(0, index); + return queryName; + } + /** + * + * @param queryIn String format of "name(param1, param2, ....)" + * @return List of params from query + * @throws QueryParseException thrown when queryIn is malformed + */ + private Map<String, String> getQueryParams(String queryIn) throws QueryParseException { + queryIn = queryIn.trim(); + String delim = ":("; + int index = queryIn.indexOf(delim); + if (index < 0) { + throw new QueryParseException("Could not parse query: '" + queryIn + "'"); + } + int indexB = queryIn.indexOf(")", index); + if (indexB < 0) { + throw new QueryParseException("Could not parse query: '" + queryIn + "'"); + } + index = index + delim.length() - 1; // We want to be pointed to end of location of + // delim + String[] temps = queryIn.substring(index + delim.length() - 1, indexB).split(","); + Map params = new HashMap<String, String>(); + for (int i = 0; i < temps.length; i++) { + params.put("param" + new Integer(i).toString(), temps[i].trim()); + } + return params; + } + + /** Will need to generalize this... right now it's specific to Errata only + * + * @param namespace namespace query lives in + * @param queryName query to run + * @param args parameters to the query + * @return List of results + */ + private List<Result> runQuery(Long sessionId, String namespace, String queryName, + Map<String, String> args) + throws SQLException, SqlMapException { + // Look up Query in DB + args.put("sessionId", sessionId.toString()); + Query query = databaseManager.getQuery(queryName); + List<Result> retval; + try { + retval = query.loadList(args); + } + finally { + query.close(); + } + return retval; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java new file mode 100644 index 0000000..0c972e0 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java @@ -0,0 +1,221 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc.handlers; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.Query; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.index.Result; +import com.redhat.satellite.search.index.QueryParseException; +import com.redhat.satellite.search.scheduler.ScheduleManager; + +import org.apache.log4j.Logger; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import redstone.xmlrpc.XmlRpcFault; + +/** + * XML-RPC handler which handles calls for indexing + * + * @version $Rev$ + */ +public class IndexHandler { + + private static Logger log = Logger.getLogger(IndexHandler.class); + private IndexManager indexManager; + private DatabaseManager databaseManager; + public static final int QUERY_ERROR = 100; + public static final int INDEX_ERROR = 200; + public static final int DB_ERROR = 300; + public static final String DEFAULT_LANG = new Locale("EN", "US").toString(); + + /** + * Constructor + * + * @param idxManager + * Search engine interface + */ + public IndexHandler(IndexManager idxManager, DatabaseManager dbMgr, + ScheduleManager schedMgr) { + indexManager = idxManager; + databaseManager = dbMgr; + } + + /** + * Search index - + * assumes English language as default language + * + * @param sessionId + * user's application session id + * @param indexName + * index to use + * @param query + * search query + * @return list of document ids as results + * @throws XmlRpcFault something bad happened + */ + public List<Result> search(long sessionId, String indexName, String query) + throws XmlRpcFault { + return search(sessionId, indexName, query, DEFAULT_LANG); + } + + /** + * Search index - + * assumes English language as default language + * + * @param sessionId + * user's application session id + * @param indexName + * index to use + * @param query + * search query + * @param isFineGrained + * if set will restrict matches to be stricter and less forgiving + * @return list of document ids as results + * @throws XmlRpcFault something bad happened + */ + public List<Result> search(long sessionId, String indexName, String query, + boolean isFineGrained) + throws XmlRpcFault { + return search(sessionId, indexName, query, DEFAULT_LANG, isFineGrained); + } + + /** + * Search index + * + * @param sessionId + * user's application session id + * @param indexName + * index to use + * @param query + * search query + * @param lang + * language + * @return list of document ids as results + * @throws XmlRpcFault something bad happened + */ + public List<Result> search(long sessionId, String indexName, String query, + String lang) throws XmlRpcFault { + return search(sessionId, indexName, query, lang, false); + } + + /** + * Search index + * + * @param sessionId + * user's application session id + * @param indexName + * index to use + * @param query + * search query + * @param lang + * language + * @param isFineGrained + * if set will restrict matches to be stricter and less forgiving + * @return list of document ids as results + * @throws XmlRpcFault something bad happened + */ + public List<Result> search(long sessionId, String indexName, String query, + String lang, boolean isFineGrained) + throws XmlRpcFault { + if (log.isDebugEnabled()) { + log.debug("IndexHandler:: searching for: " + query + ", indexName = " + + indexName + ", lang = " + lang); + } + try { + List<Result> hits = indexManager.search(indexName, query, lang, + isFineGrained); + if (indexName.compareTo("package") == 0) { + return screenHits(sessionId, indexName, hits); + } + return hits; + } + catch (IndexingException e) { + log.error("Caught exception: ", e); + throw new XmlRpcFault(INDEX_ERROR, e.getMessage()); + } + catch (QueryParseException e) { + log.error("Caught exception: ", e); + throw new XmlRpcFault(QUERY_ERROR, e.getMessage()); + } + catch (SQLException e) { + log.error("Caught exception: ", e); + throw new XmlRpcFault(DB_ERROR, e.getMessage()); + } + } + + private List<Result> screenHits(long sessionId, String indexName, + List<Result> hits) throws SQLException { + + if (hits == null || hits.size() == 0) { + if (log.isDebugEnabled()) { + log.debug("NO HITS FOUND"); + } + return Collections.EMPTY_LIST; + } + List<String> ids = new ArrayList<String>(); + for (Result pr : hits) { + ids.add(pr.getId()); + } + + Query<String> query = null; + if ("package".equals(indexName)) { + query = databaseManager.getQuery("verifyPackageVisibility"); + } + else if ("errata".equals(indexName)) { + query = databaseManager.getQuery("verifyErrataVisibility"); + } + else { + if (log.isDebugEnabled()) { + log.debug("screenHits(" + indexName + + ") no 'screening of results' performed"); + log.debug("results: " + hits); + } + return hits; + } + + Map params = new HashMap(); + params.put("session_id", sessionId); + params.put("id_list", ids); + + try { + List<String> visible = query.loadList(params); + if (log.isDebugEnabled()) { + log.debug("results: " + visible); + } + + // add the PackageResults that match the visible list + List<Result> realResults = new ArrayList<Result>(); + for (Result pr : hits) { + if (visible.contains(pr.getId())) { + realResults.add(pr); + } + } + return realResults; + } + finally { + query.close(); + } + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java new file mode 100644 index 0000000..1b88713 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc.handlers; + +import com.redhat.satellite.search.db.ResultHandler; + +import org.apache.log4j.Logger; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +/** + * Cheesy class for screening hit results based + * on user's security context + * + * @version $Rev $ + */ +class VisibilityResultsHandler implements ResultHandler { + private static Logger log = Logger + .getLogger(VisibilityResultsHandler.class); + private List<String> results = new ArrayList<String>(); + + + /** + * {@inheritDoc} + */ + public void handleRow(ResultSet rs) throws SQLException { + long id = rs.getLong(1); + results.add(String.valueOf(id)); + } + + /** + * Getter for accumulated results + * @return results + */ + List<String> getResults() { + log.warn("returning results"); + return results; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java new file mode 100644 index 0000000..15150b7 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc.handlers.tests; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.rpc.handlers.DatabaseHandler; +import com.redhat.satellite.search.scheduler.ScheduleManager; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; + +import org.apache.log4j.Logger; + +import java.util.List; + +import redstone.xmlrpc.XmlRpcFault; + + +public class DatabaseHandlerTest extends BaseTestCase { + + private static Logger log = Logger.getLogger(DatabaseHandlerTest.class); + + @Override + @SuppressWarnings("unchecked") + protected Class[] getComponentClasses() { + Class[] components = {IndexManager.class, + DatabaseManager.class, + ScheduleManager.class}; + + return TestUtil.buildComponentsList(components); + } + + public void testQuery() throws XmlRpcFault, InterruptedException { + DatabaseManager db = (DatabaseManager) + container.getComponentInstance(DatabaseManager.class); + IndexManager idx = (IndexManager) + container.getComponentInstance(IndexManager.class); + ScheduleManager schedMgr = (ScheduleManager) + container.getComponentInstance(ScheduleManager.class); + DatabaseHandler handler = new DatabaseHandler(idx, db, schedMgr); + String searchNamespace = "errata"; + String searchString = "listErrataByIssueDateRange:(2004-10-20, 2010-10-30)"; + log.info("Calling hander.search(1, " + searchNamespace + ", " + searchString + ")"); + List results = handler.search(1, searchNamespace, searchString); + log.info("results = " + results); + assertTrue(results.size() > 0); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java new file mode 100644 index 0000000..6f69a95 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.rpc.handlers.tests; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.rpc.handlers.IndexHandler; +import com.redhat.satellite.search.scheduler.ScheduleManager; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; + +import org.apache.log4j.Logger; + +import java.util.List; + +import redstone.xmlrpc.XmlRpcFault; + +public class IndexHandlerTest extends BaseTestCase { + + private static Logger log = Logger.getLogger(IndexHandlerTest.class); + + @Override + @SuppressWarnings("unchecked") + protected Class[] getComponentClasses() { + Class[] components = {IndexManager.class, + DatabaseManager.class, + ScheduleManager.class}; + + return TestUtil.buildComponentsList(components); + } + + public void testQuery() throws XmlRpcFault, InterruptedException { + // Let the indexing task do some stuff + Thread.sleep(15000); + DatabaseManager db = (DatabaseManager) + container.getComponentInstance(DatabaseManager.class); + IndexManager idx = (IndexManager) + container.getComponentInstance(IndexManager.class); + ScheduleManager schedMgr = (ScheduleManager) + container.getComponentInstance(ScheduleManager.class); + IndexHandler handler = new IndexHandler(idx, db, schedMgr); + handler.search(252437, "package", "description:package", "en"); + List results = handler.search(252437, "package", "kernel*", "en"); + log.info("kernel results 1: " + results); + results = handler.search(252437, "package", "kernel", "en"); + log.info("kernel results 2: " + results); + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java new file mode 100644 index 0000000..4a1282e --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java @@ -0,0 +1,19 @@ +package com.redhat.satellite.search.rpc.tests; + +import com.redhat.satellite.search.rpc.RpcServer; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; + +public class RpcServerTest extends BaseTestCase { + @SuppressWarnings("unchecked") + @Override + protected Class[] getComponentClasses() { + return TestUtil.buildComponentsList(RpcServer.class); + } + + public void testStopServer() { + RpcServer server = (RpcServer) + container.getComponentInstanceOfType(RpcServer.class); + server.stop(); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java new file mode 100644 index 0000000..ce19352 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java @@ -0,0 +1,202 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.scheduler; + +import com.redhat.satellite.search.config.Configuration; +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.scheduler.tasks.IndexErrataTask; +import com.redhat.satellite.search.scheduler.tasks.IndexPackagesTask; +import com.redhat.satellite.search.scheduler.tasks.IndexSnapshotTagsTask; +import com.redhat.satellite.search.scheduler.tasks.IndexServerCustomInfoTask; +import com.redhat.satellite.search.scheduler.tasks.IndexSystemsTask; +import com.redhat.satellite.search.scheduler.tasks.IndexHardwareDevicesTask; + +import org.apache.log4j.Logger; + +import org.picocontainer.Startable; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.SimpleTrigger; +import org.quartz.Trigger; +import org.quartz.impl.StdSchedulerFactory; + +import java.util.Date; + +/** + * Manages all scheduled tasks for the search server + * Right now all tasks are hardcoded -- this should be + * changed to be more config file driven + * + * @version $Rev $ + */ +public class ScheduleManager implements Startable { + private static Logger log = Logger.getLogger(ScheduleManager.class); + private Scheduler scheduler; + private DatabaseManager databaseManager; + private IndexManager indexManager; + + private final String updateIndexGroupName = "updateIndex"; + /** + * Constructor + * @param dbmgr allows ScheduleManager to access the database. + * @param idxmgr allows the ScheduleManager to access the indexer. + */ + public ScheduleManager(DatabaseManager dbmgr, IndexManager idxmgr) { + databaseManager = dbmgr; + indexManager = idxmgr; + try { + scheduler = StdSchedulerFactory.getDefaultScheduler(); + } + catch (SchedulerException e) { + throw new RuntimeException(e); + } + } + + private void scheduleJob(Scheduler sched, String name, + int mode, long interval, Class task, JobDataMap data) + throws SchedulerException { + + Trigger t = createTrigger(name, updateIndexGroupName, mode, interval); + JobDetail d = new JobDetail(name, updateIndexGroupName, task); + d.setJobDataMap(data); + sched.scheduleJob(d, t); + } + + private Trigger createTrigger(String name, String group, int mode, + long interval) { + Trigger trigger = new SimpleTrigger(name, "default", name, group, + new Date(), null, mode, interval); + trigger.setMisfireInstruction( + SimpleTrigger.MISFIRE_INSTRUCTION_RESCHEDULE_NEXT_WITH_EXISTING_COUNT); + return trigger; + } + + /** + * {@inheritDoc} + */ + public void start() { + try { + Configuration config = new Configuration(); + + long interval = config.getInt("search.schedule.interval", 300000); + log.info("ScheduleManager task interval is set to " + interval); + int mode = SimpleTrigger.REPEAT_INDEFINITELY; + if (System.getProperties().get("isTesting") != null) { + interval = 100; + mode = 0; + } + + JobDataMap jobData = new JobDataMap(); + jobData.put("indexManager", indexManager); + jobData.put("databaseManager", databaseManager); + jobData.put("configuration", new Configuration()); + + scheduleJob(scheduler, BuilderFactory.PACKAGES_TYPE, + mode, interval, + IndexPackagesTask.class, jobData); + + scheduleJob(scheduler, BuilderFactory.ERRATA_TYPE, + mode, interval, + IndexErrataTask.class, jobData); + + scheduleJob(scheduler, BuilderFactory.SERVER_TYPE, + mode, interval, + IndexSystemsTask.class, jobData); + + scheduleJob(scheduler, BuilderFactory.HARDWARE_DEVICE_TYPE, + mode, interval, + IndexHardwareDevicesTask.class, jobData); + + scheduleJob(scheduler, BuilderFactory.SNAPSHOT_TAG_TYPE, + mode, interval, + IndexSnapshotTagsTask.class, jobData); + + scheduleJob(scheduler, BuilderFactory.SERVER_CUSTOM_INFO_TYPE, + mode, interval, + IndexServerCustomInfoTask.class, jobData); + + scheduler.start(); + } + catch (SchedulerException e) { + throw new RuntimeException(e); + } + } + + /** + * {@inheritDoc} + */ + public void stop() { + try { + scheduler.shutdown(); + } + catch (SchedulerException e) { + throw new RuntimeException(e); + } + } + + private boolean isSupported(String indexName) { + if (BuilderFactory.ERRATA_TYPE.equals(indexName) || + BuilderFactory.HARDWARE_DEVICE_TYPE.equals(indexName) || + BuilderFactory.PACKAGES_TYPE.equals(indexName) || + BuilderFactory.SERVER_CUSTOM_INFO_TYPE.equals(indexName) || + BuilderFactory.SERVER_TYPE.equals(indexName) || + BuilderFactory.SNAPSHOT_TAG_TYPE.equals(indexName)) { + return true; + } + else if (BuilderFactory.DOCS_TYPE.equals(indexName)) { + log.info("Index updates for " + BuilderFactory.DOCS_TYPE + + " are not supported."); + return false; + } + log.info("Unknown index: " + indexName); + return false; + } + + /** + * Will create/schedule a trigger for the passed in indexName. + * Note: Only one trigger per indexName is allowed, if subsequent calls + * are made before the current trigger finishes completion, this request + * will be dropped. + * @param indexName + * @return + */ + public boolean triggerIndexTask(String indexName) { + if (!isSupported(indexName)) { + log.info(indexName + " is not a supported for scheduler modifications."); + return false; + } + // Define a Trigger that will fire "now" and associate it with the existing job + Trigger trigger = new SimpleTrigger("immediateTrigger-" + indexName, + "group1", new Date()); + trigger.setJobName(indexName); + trigger.setJobGroup(updateIndexGroupName); + try { + // Schedule the trigger + log.info("Scheduling trigger: " + trigger); + scheduler.scheduleJob(trigger); + } + catch (SchedulerException e) { + log.warn("Scheduling trigger: " + trigger + " failed."); + log.warn("Exception was caught: ", e); + return false; + } + return true; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java new file mode 100644 index 0000000..48e7b94 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java @@ -0,0 +1,286 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks; + +import com.ibatis.sqlmap.client.SqlMapException; +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.Query; +import com.redhat.satellite.search.db.WriteQuery; +import com.redhat.satellite.search.db.models.GenericRecord; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; + +import org.apache.log4j.Logger; +import org.apache.lucene.document.Document; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import java.sql.SQLException; + +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * GenericIndexTask + * @version $Rev$ + */ +public abstract class GenericIndexTask implements Job { + + private static Logger log = Logger.getLogger(GenericIndexTask.class); + private String lang = "en"; + /** + * {@inheritDoc} + */ + public void execute(JobExecutionContext ctx) + throws JobExecutionException { + JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); + DatabaseManager databaseManager = + (DatabaseManager)jobData.get("databaseManager"); + IndexManager indexManager = + (IndexManager)jobData.get("indexManager"); + + try { + List<GenericRecord> data = getRecords(databaseManager); + int count = 0; + log.info(super.getClass().toString() + "found [" + + data.size() + "] items to index"); + for (Iterator<GenericRecord> iter = data.iterator(); iter.hasNext();) { + GenericRecord current = iter.next(); + indexRecord(indexManager, current); + count++; + if (count == 10 || !iter.hasNext()) { + if (System.getProperties().get("isTesting") == null) { + updateLastRecord(databaseManager, current.getId()); + } + count = 0; + } + } + // + // Check to see if any records have been deleted from database, so + // we should delete from our indexes. + // + int numDel = handleDeletedRecords(databaseManager, indexManager); + log.info("Deleted " + numDel + " records from index <" + + getIndexName() + ">"); + } + catch (SQLException e) { + throw new JobExecutionException(e); + } + catch (IndexingException e) { + throw new JobExecutionException(e); + } + } + /** + * @param databaseManager + * @param sid + */ + private void updateLastRecord(DatabaseManager databaseManager, long sid) + throws SQLException { + + WriteQuery updateQuery = databaseManager.getWriterQuery(getQueryUpdateLastRecord()); + WriteQuery insertQuery = databaseManager.getWriterQuery(getQueryCreateLastRecord()); + + try { + Map params = new HashMap(); + params.put("id", sid); + params.put("last_modified", Calendar.getInstance().getTime()); + + if (updateQuery.update(params) == 0) { + insertQuery.insert(params); + } + } + finally { + try { + if (updateQuery != null) { + updateQuery.close(); + } + } + finally { + if (insertQuery != null) { + insertQuery.close(); + } + } + } + } + + /** + * @param indexManager + * @param current + */ + private void indexRecord(IndexManager indexManager, + GenericRecord data) + throws IndexingException { + + Map<String, String> attrs = getFieldMap(data); + log.info(super.getClass().toString() + " Indexing object: " + + data.getId() + ": " + attrs.toString()); + DocumentBuilder pdb = BuilderFactory.getBuilder(getIndexName()); + Document doc = pdb.buildDocument(new Long(data.getId()), attrs); + indexManager.addUniqueToIndex(getIndexName(), doc, getUniqueFieldId(), + lang); + } + + + /** + * @param databaseManager + * @return + */ + private List<GenericRecord> getRecords(DatabaseManager databaseManager) + throws SQLException { + // What was the last object id we indexed? + List<GenericRecord> retval = null; + Query<Long> query = databaseManager.getQuery(getQueryLastRecord()); + Long sid = null; + try { + sid = query.load(); + } + finally { + query.close(); + } + if (sid == null) { + sid = new Long(0); + } + // When was the last time we ran the indexing of servers? + Query<Date> queryLast = databaseManager.getQuery(getQueryLastIndexDate()); + Date lastRun = null; + try { + lastRun = queryLast.load(); + } + finally { + queryLast.close(); + } + if (lastRun == null) { + lastRun = new Date(0); + } + // Lookup what objects have not been indexed, or need to be reindexed. + Query<GenericRecord> srvrQuery = databaseManager.getQuery( + getQueryRecordsToIndex()); + try { + Map params = new HashMap(); + params.put("id", sid); + params.put("last_modified", lastRun); + log.info("GenericIndexTask<" + super.getClass().toString() + + " last processed id = " + sid + ", lastRun was " + lastRun); + retval = srvrQuery.loadList(params); + log.info("GenericIndexTask<" + super.getClass().toString() + + " number of results returned = " + retval.size()); + } + finally { + srvrQuery.close(); + } + return retval; + } + + /** + * Will determine if any records have been deleted from the DB, then will + * delete those records from the lucene index. + * @return number of deleted records + */ + protected int handleDeletedRecords(DatabaseManager databaseManager, + IndexManager indexManager) + throws SQLException { + List<Long> ids = null; + Query<Long> query = null; + String uniqField = null; + String indexName = null; + HashSet<String> idSet = null; + try { + query = databaseManager.getQuery(getQueryAllIds()); + ids = query.loadList(Collections.EMPTY_MAP); + if ((ids == null) || (ids.size() == 0)) { + log.info("Got back no data from '" + getQueryAllIds() + "'"); + log.info("Skipping the handleDeletedRecords() method"); + return 0; + } + idSet = new HashSet(); + for (Long num : ids) { + idSet.add(num.toString()); + } + uniqField = getUniqueFieldId(); + indexName = getIndexName(); + } + catch (SqlMapException e) { + e.printStackTrace(); + log.info("Error with 'getQueryAllIds()' on " + + super.getClass().toString()); + //just print the warning so we know and skip this method. + return 0; + } + finally { + if (query != null) { + query.close(); + } + } + return indexManager.deleteRecordsNotInList(idSet, indexName, uniqField); + } + + /** + * + * @param data fully populated DTO object + * @return map which represents the fields to index along with their values + * @throws ClassCastException when data isn't castable to the intended DTO + */ + protected abstract Map<String, String> getFieldMap(GenericRecord data) + throws ClassCastException; + /** + * + * @return the index name + */ + public abstract String getIndexName(); + /** + * @return the Document field name which represents the unique id for this data + */ + public abstract String getUniqueFieldId(); + /** + * + * @return name of query which shows the last record indexed + */ + protected abstract String getQueryLastRecord(); + /** + * + * @return name of query which will update the last record indexed + */ + protected abstract String getQueryUpdateLastRecord(); + /** + * + * @return name of query which will create the last record indexed + */ + protected abstract String getQueryCreateLastRecord(); + /** + * + * @return name of query which will give back records to be indexed + */ + protected abstract String getQueryRecordsToIndex(); + /** + * + * @return name of query which will show the date this task last ran + */ + protected abstract String getQueryLastIndexDate(); + + /** + * @return name of the query which will return all current ids. + */ + protected abstract String getQueryAllIds(); +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java new file mode 100644 index 0000000..01599a4 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java @@ -0,0 +1,140 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks; + +import com.redhat.satellite.search.config.Configuration; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.scheduler.tasks.crawl.WebCrawl; + +import org.apache.log4j.Logger; + +import org.quartz.JobDataMap; +import org.quartz.StatefulJob; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import java.io.File; +import java.io.IOException; + + +/** + * Task to index help documents + * + * @version $Rev: $ + */ +public class IndexDocumentsTask implements StatefulJob { + // We do _not_ want this task to run concurrently with itself, + // therefore using StatefulJob. + private static Logger log = Logger.getLogger(IndexDocumentsTask.class); + public static final String TASK_REINDEX = new String("TASK.RE-INDEX"); + public static final String TASK_COMPLETE = new String("TASK.COMPLETE"); + + + /** + * {@inheritDoc} + */ + public void execute(JobExecutionContext ctx) + throws JobExecutionException { + JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); + IndexManager indexManager = (IndexManager) jobData.get("indexManager"); + + String indexWorkDir = indexManager.getIndexWorkDir(); + if ((indexWorkDir == null) || (indexWorkDir.compareTo("") == 0)) { + throw new JobExecutionException("indexWorkDir invalid"); + } + File docsIndexDir = new File(indexWorkDir + File.separator + + IndexManager.DOCS_INDEX_NAME); + File indexCheck = new File(docsIndexDir, TASK_REINDEX); + File success = new File(docsIndexDir, TASK_COMPLETE); + + boolean reindex = false; + + log.info("task running"); + + if (!docsIndexDir.exists()) { + // first time running, the index needs to be created from scratch + log.info("Creating directory: " + docsIndexDir.getPath()); + if (!docsIndexDir.mkdirs()) { + throw new JobExecutionException("Unable to create dir: " + + docsIndexDir.getPath()); + } + reindex = true; + } + else { + // Will reindex if a TASK.RE-INDEX is present or + // we don't see TASK.COMPLETE + log.info("docsIndexDir<" + docsIndexDir + "> exists"); + if (indexCheck.exists()) { + log.info("Found (" + indexCheck.getPath() + + "), Index needs to be updated."); + reindex = true; + } + else if (!success.exists()) { + log.info("Index exists, but it appears to be incomplete, missing " + + success.getPath()); + reindex = true; + } + } + + if (reindex) { + //We need to communicate index is incomplete by removing the success file + + if (success.exists()) { + if (!success.delete()) { + throw new JobExecutionException("Unable to delete file: " + + success.getPath()); + } + } + if ((indexCheck != null) && (indexCheck.exists())) { + // cleanup the file which triggered the re-index + if (!indexCheck.delete()) { + throw new JobExecutionException("Unable to delete file: " + + indexCheck.getPath()); + } + } + try { + Configuration config = (Configuration)jobData.get("configuration"); + if (!reIndexDocs(docsIndexDir.getPath(), config)) { + log.error("Failed reindexing documents to : "); + throw new JobExecutionException("Reindexing of Documents failed."); + } + //Mark that docs have been indexed and are ready to be used. + success.createNewFile(); + } + catch (IOException e) { + throw new JobExecutionException(e); + } + } + } + + protected boolean reIndexDocs(String docsIndexDir, Configuration config) + throws IOException { + + int threads = config.getInt("search.nutch.threads", 10); + int depth = config.getInt("search.nutch.depth", 5); + String inputUrlFile = config.getString("search.nutch.inputUrlFile", "nutch/urls"); + String tmpCrawlDir = config.getString("search.nutch.tmpDir", "/tmp/crawl_output"); + + WebCrawl wCrawl = new WebCrawl(); + wCrawl.setDepth(depth); + wCrawl.setInputUrlFile(inputUrlFile); + wCrawl.setOutputIndexDir(docsIndexDir); + wCrawl.setThreads(threads); + wCrawl.setTmpCrawlDir(tmpCrawlDir); + + return wCrawl.crawl(); + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java new file mode 100644 index 0000000..1c3ef6b --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java @@ -0,0 +1,178 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.Query; +import com.redhat.satellite.search.db.WriteQuery; +import com.redhat.satellite.search.db.models.Errata; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; + +import org.apache.log4j.Logger; +import org.apache.lucene.document.Document; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import java.sql.SQLException; +import java.util.Calendar; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + + +/** + * IndexErrataTask + * @version $Rev$ + */ +public class IndexErrataTask implements Job { + + private static Logger log = Logger.getLogger(IndexErrataTask.class); + private String lang = "en"; + /** + * {@inheritDoc} + */ + public void execute(JobExecutionContext ctx) + throws JobExecutionException { + JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); + DatabaseManager databaseManager = + (DatabaseManager)jobData.get("databaseManager"); + IndexManager indexManager = + (IndexManager)jobData.get("indexManager"); + + try { + + List<Errata> errata = getErrata(databaseManager); + int count = 0; + log.info("found [" + errata.size() + "] errata to index"); + for (Iterator<Errata> iter = errata.iterator(); iter.hasNext();) { + Errata current = iter.next(); + indexErrata(indexManager, current); + count++; + if (count == 10 || !iter.hasNext()) { + if (System.getProperties().get("isTesting") == null) { + updateLastErrataId(databaseManager, current.getId()); + } + count = 0; + } + } + } + catch (SQLException e) { + throw new JobExecutionException(e); + } + catch (IndexingException e) { + throw new JobExecutionException(e); + } + } + /** + * @param databaseManager + * @param sid + */ + private void updateLastErrataId(DatabaseManager databaseManager, long eid) + throws SQLException { + + WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastErrata"); + WriteQuery insertQuery = databaseManager.getWriterQuery("createLastErrata"); + + try { + Map params = new HashMap(); + params.put("id", eid); + params.put("last_modified", Calendar.getInstance().getTime()); + if (updateQuery.update(params) == 0) { + insertQuery.insert(params); + } + } + finally { + try { + if (updateQuery != null) { + updateQuery.close(); + } + } + finally { + if (insertQuery != null) { + insertQuery.close(); + } + } + } + } + + /** + * @param indexManager + * @param current + */ + private void indexErrata(IndexManager indexManager, Errata errata) + throws IndexingException { + + Map<String, String> attrs = new HashMap<String, String>(); + attrs.put("id", new Long(errata.getId()).toString()); + attrs.put("advisory", errata.getAdvisory()); + attrs.put("advisoryType", errata.getAdvisoryType()); + attrs.put("advisoryName", errata.getAdvisoryName()); + attrs.put("advisoryRel", new Long(errata.getAdvisoryRel()).toString()); + attrs.put("product", errata.getProduct()); + attrs.put("description", errata.getDescription()); + attrs.put("synopsis", errata.getSynopsis()); + attrs.put("topic", errata.getTopic()); + attrs.put("solution", errata.getSolution()); + attrs.put("issueDate", errata.getIssueDate()); + attrs.put("updateDate", errata.getUpdateDate()); + attrs.put("notes", errata.getNotes()); + attrs.put("orgId", errata.getOrgId()); + attrs.put("created", errata.getCreated()); + attrs.put("modified", errata.getModified()); + attrs.put("lastModified", errata.getLastModified()); + attrs.put("name", errata.getAdvisory()); + + log.info("Indexing errata: " + errata.getId() + ": " + attrs.toString()); + DocumentBuilder edb = BuilderFactory.getBuilder(BuilderFactory.ERRATA_TYPE); + Document doc = edb.buildDocument(new Long(errata.getId()), attrs); + indexManager.addToIndex("errata", doc, lang); + } + + /** + * @param databaseManager + * @return + */ + private List<Errata> getErrata(DatabaseManager databaseManager) + throws SQLException { + + List<Errata> retval = null; + Query<Long> query = databaseManager.getQuery("getLastErrataId"); + Long eid = null; + try { + eid = query.load(); + } + finally { + query.close(); + } + if (eid == null) { + eid = new Long(0); + } + Query<Errata> errataQuery = databaseManager.getQuery("listErrataFromId"); + try { + retval = errataQuery.loadList(eid); + } + finally { + errataQuery.close(); + } + return retval; + } + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java new file mode 100644 index 0000000..8960d94 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java @@ -0,0 +1,117 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks; + +import com.redhat.satellite.search.db.models.GenericRecord; +import com.redhat.satellite.search.db.models.HardwareDevice; +import com.redhat.satellite.search.index.builder.BuilderFactory; + +import org.apache.log4j.Logger; + +import java.util.HashMap; +import java.util.Map; + +/** + * IndexHardwareDevicesTask + * @version $Rev$ + */ +public class IndexHardwareDevicesTask extends GenericIndexTask { + + private static Logger log = Logger.getLogger(IndexHardwareDevicesTask.class); + + /** + * {@inheritDoc} + */ + @Override + protected Map<String, String> getFieldMap(GenericRecord data) + throws ClassCastException { + HardwareDevice dev = (HardwareDevice)data; + Map<String, String> attrs = new HashMap<String, String>(); + attrs.put("serverId", new Long(dev.getServerId()).toString()); + attrs.put("classInfo", dev.getClassInfo()); + attrs.put("bus", dev.getBus()); + attrs.put("detached", new Long(dev.getDetached()).toString()); + attrs.put("device", dev.getDevice()); + attrs.put("driver", dev.getDriver()); + attrs.put("description", dev.getDescription()); + attrs.put("vendorId", dev.getVendorId()); + attrs.put("deviceId", dev.getDeviceId()); + attrs.put("subVendorId", dev.getSubVendorId()); + attrs.put("subDeviceId", dev.getSubDeviceId()); + attrs.put("pciType", new Long(dev.getPciType()).toString()); + return attrs; + } + + /** + * {@inheritDoc} + */ + @Override + public String getIndexName() { + return BuilderFactory.HARDWARE_DEVICE_TYPE; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryCreateLastRecord() { + return "createLastHardwareDevice"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastRecord() { + return "getLastHardwareDeviceId"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastIndexDate() { + return "getLastHardwareDeviceIndexRun"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryRecordsToIndex() { + return "getHardwareDeviceById"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryUpdateLastRecord() { + return "updateLastHardwareDevice"; + } + + /** + * {@inheritDoc} + */ + public String getUniqueFieldId() { + return "id"; + } + /** + * {@inheritDoc} + */ + public String getQueryAllIds() { + return "queryAllHwDeviceIds"; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java new file mode 100644 index 0000000..7c657e6 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java @@ -0,0 +1,163 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ + +package com.redhat.satellite.search.scheduler.tasks; + +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.Query; +import com.redhat.satellite.search.db.WriteQuery; +import com.redhat.satellite.search.db.models.RhnPackage; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.index.builder.DocumentBuilder; + +import org.apache.log4j.Logger; +import org.apache.lucene.document.Document; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import java.sql.SQLException; +import java.util.Calendar; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Task to index package information + * + * @version $Rev$ + */ +public class IndexPackagesTask implements Job { + + private static Logger log = Logger.getLogger(IndexPackagesTask.class); + private String lang = "en"; + /** + * {@inheritDoc} + */ + public void execute(JobExecutionContext ctx) throws JobExecutionException { + JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); + DatabaseManager databaseManager = + (DatabaseManager)jobData.get("databaseManager"); + IndexManager indexManager = + (IndexManager)jobData.get("indexManager"); + + try { + if (System.getProperties().get("isTesting") != null) { + cleanLastPackage(databaseManager); + } + List<RhnPackage> packages = getPackages(databaseManager); + int count = 0; + log.info("found [" + packages.size() + "] packages to index"); + for (Iterator<RhnPackage> iter = packages.iterator(); iter.hasNext();) { + RhnPackage current = iter.next(); + indexPackage(indexManager, current); + count++; + if (count == 10 || !iter.hasNext()) { + if (System.getProperties().get("isTesting") == null) { + updateLastPackageId(databaseManager, current.getId()); + } + count = 0; + } + } + } + catch (SQLException e) { + throw new JobExecutionException(e); + } + catch (IndexingException e) { + throw new JobExecutionException(e); + } + } + + private void cleanLastPackage(DatabaseManager databaseManager) throws SQLException { + WriteQuery query = null; + try { + query = databaseManager.getWriterQuery("deleteLastPackage"); + query.delete(null); + } + finally { + query.close(); + } + } + + private void indexPackage(IndexManager indexManager, RhnPackage pkg) + throws IndexingException { + Map<String, String> attrs = new HashMap<String, String>(); + attrs.put("name", pkg.getName()); + attrs.put("version", pkg.getVersion()); + attrs.put("release", pkg.getRelease()); + attrs.put("filename", pkg.getFileName()); + attrs.put("description", pkg.getDescription()); + attrs.put("summary", pkg.getSummary()); + attrs.put("arch", pkg.getArch()); + log.info("Indexing package: " + pkg.getId() + ": " + attrs.toString()); + DocumentBuilder pdb = BuilderFactory.getBuilder(BuilderFactory.PACKAGES_TYPE); + Document doc = pdb.buildDocument(new Long(pkg.getId()), attrs); + indexManager.addToIndex("package", doc, lang); + } + + private void updateLastPackageId(DatabaseManager databaseManager, + Long packageId) throws SQLException { + WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastPackage"); + WriteQuery insertQuery = databaseManager.getWriterQuery("createLastPackage"); + try { + Map params = new HashMap(); + params.put("id", packageId); + params.put("last_modified", Calendar.getInstance().getTime()); + if (updateQuery.update(params) == 0) { + insertQuery.insert(params); + } + } + finally { + try { + if (updateQuery != null) { + updateQuery.close(); + } + } + finally { + if (insertQuery != null) { + insertQuery.close(); + } + } + } + } + + private List<RhnPackage> getPackages(DatabaseManager databaseManager) + throws SQLException { + List<RhnPackage> retval = null; + Query<Long> query = databaseManager.getQuery("getLastPackageId"); + Long packageId = null; + try { + packageId = query.load(); + } + finally { + query.close(); + } + if (packageId == null) { + packageId = new Long(0); + } + Query<RhnPackage> pkgQuery = databaseManager.getQuery("listPackagesFromId"); + try { + retval = pkgQuery.loadList(packageId); + } + finally { + pkgQuery.close(); + } + return retval; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java new file mode 100644 index 0000000..ef7b366 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java @@ -0,0 +1,167 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks; + +import com.ibatis.sqlmap.client.SqlMapException; +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.Query; +import com.redhat.satellite.search.db.models.GenericRecord; +import com.redhat.satellite.search.db.models.ServerCustomInfo; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.builder.BuilderFactory; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; + +/** + * IndexServerCustomInfoTask + * @version $Rev$ + * + */ +public class IndexServerCustomInfoTask extends GenericIndexTask { + private static Logger log = Logger.getLogger(IndexServerCustomInfoTask.class); + + /** + * Will determine if any records have been deleted from the DB, then will + * delete those records from the lucene index. + * @return number of deleted records + */ + protected int handleDeletedRecords(DatabaseManager databaseManager, + IndexManager indexManager) + throws SQLException { + List<ServerCustomInfo> ids = null; + Query<ServerCustomInfo> query = null; + String uniqField = null; + String indexName = null; + HashSet<String> idSet = null; + try { + query = databaseManager.getQuery(getQueryAllIds()); + ids = query.loadList(Collections.EMPTY_MAP); + if ((ids == null) || (ids.size() == 0)) { + log.info("Got back no data from '" + getQueryAllIds() + "'"); + log.info("Skipping the handleDeletedRecords() method"); + return 0; + } + idSet = new HashSet(); + for (ServerCustomInfo scInfo : ids) { + idSet.add(scInfo.getUniqId()); + } + uniqField = getUniqueFieldId(); + indexName = getIndexName(); + } + catch (SqlMapException e) { + e.printStackTrace(); + log.info("Error with 'getQueryAllIds()' on " + + super.getClass().toString()); + //just print the warning so we know and skip this method. + return 0; + } + finally { + if (query != null) { + query.close(); + } + } + return indexManager.deleteRecordsNotInList(idSet, indexName, uniqField); + } + + + + + /** + * {@inheritDoc} + */ + @Override + protected Map<String, String> getFieldMap(GenericRecord data) + throws ClassCastException { + ServerCustomInfo scInfo = (ServerCustomInfo)data; + Map<String, String> attrs = new HashMap<String, String>(); + attrs.put("uniqId", scInfo.getUniqId()); + attrs.put("serverId", new Long(scInfo.getServerId()).toString()); + attrs.put("value", scInfo.getValue()); + attrs.put("label", scInfo.getLabel()); + attrs.put("createdBy", new Long(scInfo.getCreatedBy()).toString()); + attrs.put("lastModifiedBy", new Long(scInfo.getLastModifiedBy()).toString()); + attrs.put("created", scInfo.getCreated()); + attrs.put("modified", scInfo.getModified()); + return attrs; + } + + /** + * {@inheritDoc} + */ + @Override + public String getIndexName() { + return BuilderFactory.SERVER_CUSTOM_INFO_TYPE; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryCreateLastRecord() { + return "createLastServerCustomInfo"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastRecord() { + return "getLastServerCustomInfoId"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastIndexDate() { + return "getLastServerCustomInfoIndexRun"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryRecordsToIndex() { + return "getServerCustomInfoByIdOrDate"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryUpdateLastRecord() { + return "updateLastServerCustomInfo"; + } + + /** + * {@inheritDoc} + */ + public String getUniqueFieldId() { + return "uniqId"; + } + /** + * {@inheritDoc} + */ + public String getQueryAllIds() { + return "queryAllServerCustomInfoIds"; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java new file mode 100644 index 0000000..611f1c6 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java @@ -0,0 +1,110 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks; + +import com.redhat.satellite.search.db.models.GenericRecord; +import com.redhat.satellite.search.db.models.SnapshotTag; +import com.redhat.satellite.search.index.builder.BuilderFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * IndexSnapshotTagsTask + * @version $Rev$ + * + */ +public class IndexSnapshotTagsTask extends GenericIndexTask { + + /** + * {@inheritDoc} + */ + @Override + protected Map<String, String> getFieldMap(GenericRecord data) + throws ClassCastException { + SnapshotTag sTag = (SnapshotTag)data; + Map<String, String> attrs = new HashMap<String, String>(); + attrs.put("snapshotId", new Long(sTag.getSnapshotId()).toString()); + attrs.put("tagNameId", new Long(sTag.getTagNameId()).toString()); + attrs.put("serverId", new Long(sTag.getServerId()).toString()); + attrs.put("orgId", new Long(sTag.getOrgId()).toString()); + attrs.put("name", sTag.getName()); + attrs.put("created", sTag.getCreated()); + attrs.put("modified", sTag.getModified()); + return attrs; + } + + /** + * {@inheritDoc} + */ + @Override + public String getIndexName() { + return BuilderFactory.SNAPSHOT_TAG_TYPE; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryCreateLastRecord() { + return "createLastSnapshotTag"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastRecord() { + return "getLastSnapshotTagId"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastIndexDate() { + return "getLastSnapshotTagIndexRun"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryRecordsToIndex() { + return "getSnapshotTagByIdOrDate"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryUpdateLastRecord() { + return "updateLastSnapshotTag"; + } + + /** + * {@inheritDoc} + */ + public String getUniqueFieldId() { + return "tagNameId"; + } + + /** + * {@inheritDoc} + */ + public String getQueryAllIds() { + return "queryAllSnapshotTagIds"; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java new file mode 100644 index 0000000..4cb071b --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java @@ -0,0 +1,155 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks; + + +import com.redhat.satellite.search.db.models.GenericRecord; +import com.redhat.satellite.search.db.models.Server; +import com.redhat.satellite.search.index.builder.BuilderFactory; + +import java.util.HashMap; + +import java.util.Map; + + +// Main tasks: +// 1) Index new systems, i.e. system id is greater than last recorded system id indexed. +// 2) Update the existing index of systems which have been modified +// 3) Remove systems which have been deleted from the system. + +/** + * IndexSystemsTask + * @version $Rev$ + */ +public class IndexSystemsTask extends GenericIndexTask { + /** + * {@inheritDoc} + */ + @Override + protected Map<String, String> getFieldMap(GenericRecord data) + throws ClassCastException { + Server srvr = (Server)data; + + Map<String, String> attrs = new HashMap<String, String>(); + attrs.put("id", new Long(srvr.getId()).toString()); + attrs.put("system_id", new Long(srvr.getId()).toString()); + attrs.put("name", srvr.getName()); + attrs.put("description", srvr.getDescription()); + attrs.put("info", srvr.getInfo()); + attrs.put("runningKernel", srvr.getRunningKernel()); + attrs.put("machine", srvr.getMachine()); + attrs.put("rack", srvr.getRack()); + attrs.put("room", srvr.getRoom()); + attrs.put("building", srvr.getBuilding()); + attrs.put("address1", srvr.getAddress1()); + attrs.put("address2", srvr.getAddress2()); + attrs.put("city", srvr.getCity()); + attrs.put("state", srvr.getState()); + attrs.put("country", srvr.getCountry()); + attrs.put("hostname", srvr.getHostname()); + attrs.put("ipaddr", srvr.getIpaddr()); + attrs.put("dmiVendor", srvr.getDmiVendor()); + attrs.put("dmiSystem", srvr.getDmiSystem()); + attrs.put("dmiProduct", srvr.getDmiProduct()); + attrs.put("dmiBiosVendor", srvr.getDmiBiosVendor()); + attrs.put("dmiBiosVersion", srvr.getDmiBiosVersion()); + attrs.put("dmiBiosRelease", srvr.getDmiBiosRelease()); + attrs.put("dmiAsset", srvr.getDmiAsset()); + attrs.put("dmiBoard", srvr.getDmiBoard()); + attrs.put("cpuBogoMIPs", srvr.getCpuBogoMIPS()); + attrs.put("cpuCache", srvr.getCpuCache()); + attrs.put("cpuFamily", srvr.getCpuFamily()); + attrs.put("cpuMhz", srvr.getCpuMhz()); + attrs.put("cpuStepping", srvr.getCpuStepping()); + attrs.put("cpuFlags", srvr.getCpuFlags()); + attrs.put("cpuModel", srvr.getCpuModel()); + attrs.put("cpuVersion", srvr.getCpuVersion()); + attrs.put("cpuVendor", srvr.getCpuVendor()); + if (srvr.getCpuNumberOfCpus() != null) { + attrs.put("cpuNumberOfCpus", srvr.getCpuNumberOfCpus().toString()); + } + attrs.put("cpuAcpiVersion", srvr.getCpuAcpiVersion()); + attrs.put("cpuApic", srvr.getCpuApic()); + attrs.put("cpuApmVersion", srvr.getCpuApmVersion()); + attrs.put("cpuChipset", srvr.getCpuChipset()); + attrs.put("checkin", srvr.getCheckin()); + attrs.put("registered", srvr.getRegistered()); + attrs.put("ram", srvr.getRam()); + attrs.put("swap", srvr.getSwap()); + return attrs; + } + + /** + * {@inheritDoc} + */ + @Override + public String getIndexName() { + return BuilderFactory.SERVER_TYPE; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryCreateLastRecord() { + return "createLastServer"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastRecord() { + return "getLastServerId"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryLastIndexDate() { + return "getLastServerIndexRun"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryRecordsToIndex() { + return "getServerByIdOrDate"; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getQueryUpdateLastRecord() { + return "updateLastServer"; + } + + /** + * {@inheritDoc} + */ + public String getUniqueFieldId() { + return "id"; + } + + /** + * {@inheritDoc} + */ + public String getQueryAllIds() { + return "queryAllServerIds"; + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java new file mode 100644 index 0000000..9352f3c --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java @@ -0,0 +1,185 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks.tests; + +import com.redhat.satellite.search.config.Configuration; +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.WriteQuery; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.scheduler.tasks.IndexErrataTask; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; + +import org.apache.lucene.document.Document; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.JobExecutionContext; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.SimpleTrigger; +import org.quartz.Trigger; +import org.quartz.TriggerListener; +import org.quartz.impl.StdSchedulerFactory; + +import org.apache.log4j.Logger; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + + +/** + * IndexErrataTaskTest + * @version $Rev$ + */ +public class IndexErrataTaskTest extends BaseTestCase { + private static Logger log = Logger.getLogger(IndexErrataTaskTest.class); + private Scheduler scheduler; + + public void tearDown() throws Exception { + DatabaseManager databaseManager = (DatabaseManager) + container.getComponentInstanceOfType(DatabaseManager.class); + WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastErrata"); + Map params = new HashMap(); + params.put("id", 0L); + params.put("last_modified", new Date(0)); + updateQuery.update(params); + updateQuery.close(); + if (scheduler != null) { + scheduler.shutdown(false); + } + super.tearDown(); + } + + public void testExecute() throws InterruptedException, SchedulerException { + + DatabaseManager databaseManager = (DatabaseManager) + container.getComponentInstanceOfType(DatabaseManager.class); + + IndexManager indexManager = (IndexManager) + container.getComponentInstanceOfType(IndexManager.class); + + try { + scheduler = StdSchedulerFactory.getDefaultScheduler(); + } + catch (SchedulerException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + Trigger trigger = new SimpleTrigger("index", + "default", + "index", + "default", + new Date(), + null, + 0, + 100); + trigger.setMisfireInstruction( + SimpleTrigger.MISFIRE_INSTRUCTION_FIRE_NOW); + JobDetail detail = new JobDetail("index", "default", IndexErrataTask.class); + JobDataMap jobData = new JobDataMap(); + jobData.put("indexManager", indexManager); + jobData.put("databaseManager", databaseManager); + detail.setJobDataMap(jobData); + try { + scheduler.addTriggerListener(new TestTrigger()); + scheduler.scheduleJob(detail, trigger); + } + catch (SchedulerException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + try { + + scheduler.start(); + } + catch (SchedulerException e) { + System.out.println("woo hoo"); + } + Thread.sleep(1000); + if(!scheduler.isShutdown()) { + scheduler.shutdown(true); + } + System.out.println("running"); + } + + /** + * {@inheritDoc} + */ + @Override + protected Class[] getComponentClasses() { + Class[] comps = {DatabaseManager.class, TestIndexManager.class}; + return TestUtil.buildComponentsList(comps); + } + + public static class TestIndexManager extends IndexManager { + + @Override + public void addToIndex(String indexName, Document doc, String lang) throws IndexingException { + assertNotNull(doc); + assertNotNull(doc.getField("advisory")); + assertNotNull(doc.getField("advisory").stringValue()); + log.info("idx[" + indexName + "] doc [" + doc.toString() + "]"); + } + + /** + * @param config + */ + public TestIndexManager(Configuration config) { + super(config); + } + + } + + public static class TestTrigger implements TriggerListener { + + /** + * {@inheritDoc} + */ + public String getName() { + return "test trigger"; + } + + /** + * {@inheritDoc} + */ + public void triggerComplete(Trigger trigger, JobExecutionContext context, int triggerInstructionCode) { + System.out.println("**************************** completed"); + } + + /** + * {@inheritDoc} + */ + public void triggerFired(Trigger trigger, JobExecutionContext context) { + System.out.println("**************************** fired"); + } + + /** + * {@inheritDoc} + */ + public void triggerMisfired(Trigger trigger) { + System.out.println("**************************** misfire"); + } + + /** + * {@inheritDoc} + */ + public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) { + return false; + } + + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java new file mode 100644 index 0000000..2f43854 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java @@ -0,0 +1,186 @@ +/** + * Copyright (c) 2008 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.satellite.search.scheduler.tasks.tests; + +import com.redhat.satellite.search.config.Configuration; +import com.redhat.satellite.search.db.DatabaseManager; +import com.redhat.satellite.search.db.WriteQuery; +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.scheduler.tasks.IndexSystemsTask; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; + +import org.apache.lucene.document.Document; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.JobExecutionContext; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.SimpleTrigger; +import org.quartz.Trigger; +import org.quartz.TriggerListener; +import org.quartz.impl.StdSchedulerFactory; + +import org.apache.log4j.Logger; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + + +/** + * IndexSystemsTaskTest + * @version $Rev$ + */ +public class IndexSystemsTaskTest extends BaseTestCase { + private static Logger log = Logger.getLogger(IndexSystemsTaskTest.class); + private Scheduler scheduler; + + public void tearDown() throws Exception { + DatabaseManager databaseManager = (DatabaseManager) + container.getComponentInstanceOfType(DatabaseManager.class); + WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastServer"); + Map params = new HashMap(); + params.put("id", 0L); + params.put("last_modified", new Date(0)); + updateQuery.update(params); + updateQuery.close(); + scheduler.shutdown(false); + + super.tearDown(); + } + + public void testExecute() throws InterruptedException, SchedulerException { + + DatabaseManager databaseManager = (DatabaseManager) + container.getComponentInstanceOfType(DatabaseManager.class); + + IndexManager indexManager = (IndexManager) + container.getComponentInstanceOfType(IndexManager.class); + + try { + scheduler = StdSchedulerFactory.getDefaultScheduler(); + } + catch (SchedulerException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + Trigger trigger = new SimpleTrigger("index", + "default", + "index", + "default", + new Date(), + null, + 0, + 100); + trigger.setMisfireInstruction( + SimpleTrigger.MISFIRE_INSTRUCTION_FIRE_NOW); + JobDetail detail = new JobDetail("index", "default", IndexSystemsTask.class); + JobDataMap jobData = new JobDataMap(); + jobData.put("indexManager", indexManager); + jobData.put("databaseManager", databaseManager); + detail.setJobDataMap(jobData); + try { + scheduler.addTriggerListener(new TestTrigger()); + scheduler.scheduleJob(detail, trigger); + } + catch (SchedulerException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + try { + + scheduler.start(); + } + catch (SchedulerException e) { + System.out.println("woo hoo"); + } + Thread.sleep(1000); + if(!scheduler.isShutdown()) { + scheduler.shutdown(true); + } + System.out.println("running"); + } + + /** + * {@inheritDoc} + */ + @Override + protected Class[] getComponentClasses() { + Class[] comps = {DatabaseManager.class, TestIndexManager.class}; + return TestUtil.buildComponentsList(comps); + } + + public static class TestIndexManager extends IndexManager { + + @Override + public void addToIndex(String indexName, Document doc, String lang) throws IndexingException { + assertNotNull(doc); + assertNotNull(doc.getField("id")); + assertNotNull(doc.getField("name").stringValue()); + assertNotNull(doc.getField("description").stringValue()); + assertNotNull(doc.getField("info").stringValue()); + log.info("idx[" + indexName + "] doc [" + doc.toString() + "]"); + } + + /** + * @param config + */ + public TestIndexManager(Configuration config) { + super(config); + } + + } + + public static class TestTrigger implements TriggerListener { + + /** + * {@inheritDoc} + */ + public String getName() { + return "test trigger"; + } + + /** + * {@inheritDoc} + */ + public void triggerComplete(Trigger trigger, JobExecutionContext context, int triggerInstructionCode) { + System.out.println("**************************** completed"); + } + + /** + * {@inheritDoc} + */ + public void triggerFired(Trigger trigger, JobExecutionContext context) { + System.out.println("**************************** fired"); + } + + /** + * {@inheritDoc} + */ + public void triggerMisfired(Trigger trigger) { + System.out.println("**************************** misfire"); + } + + /** + * {@inheritDoc} + */ + public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) { + return false; + } + + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java new file mode 100644 index 0000000..382c3c7 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java @@ -0,0 +1,48 @@ +package com.redhat.satellite.search.scheduler.tests; + +import com.redhat.satellite.search.index.IndexManager; +import com.redhat.satellite.search.index.IndexingException; +import com.redhat.satellite.search.index.QueryParseException; +import com.redhat.satellite.search.index.Result; +import com.redhat.satellite.search.index.builder.BuilderFactory; +import com.redhat.satellite.search.scheduler.ScheduleManager; +import com.redhat.satellite.search.tests.BaseTestCase; +import com.redhat.satellite.search.tests.TestUtil; + +import java.util.List; + +public class ScheduleManagerTest extends BaseTestCase { + @SuppressWarnings("unchecked") + @Override + protected Class[] getComponentClasses() { + return TestUtil.buildComponentsList(ScheduleManager.class); + } + + public void testIndexing() throws IndexingException, QueryParseException { + try { + Thread.sleep(30000); + IndexManager mgr = (IndexManager) + container.getComponentInstance(IndexManager.class); + List<Result>hits = mgr.search("package", "description:package", + "en"); + assertTrue(hits.size() > 0); + } + catch (InterruptedException e) { + return; + } + } + + public void testTriggerIndexTask() { + ScheduleManager sm = new ScheduleManager(null, null); + assertTrue(sm.triggerIndexTask(BuilderFactory.ERRATA_TYPE)); + assertTrue(sm.triggerIndexTask(BuilderFactory.HARDWARE_DEVICE_TYPE)); + assertTrue(sm.triggerIndexTask(BuilderFactory.PACKAGES_TYPE)); + assertTrue(sm.triggerIndexTask(BuilderFactory.SERVER_CUSTOM_INFO_TYPE)); + assertTrue(sm.triggerIndexTask(BuilderFactory.SERVER_TYPE)); + assertTrue(sm.triggerIndexTask(BuilderFactory.SNAPSHOT_TAG_TYPE)); + assertFalse(sm.triggerIndexTask(BuilderFactory.DOCS_TYPE)); + assertFalse(sm.triggerIndexTask(null)); + assertFalse(sm.triggerIndexTask("biteme")); + assertFalse(sm.triggerIndexTask("")); + } +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/BaseTestCase.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/BaseTestCase.java new file mode 100644 index 0000000..15e6e90 --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/BaseTestCase.java @@ -0,0 +1,33 @@ +package com.redhat.satellite.search.tests; + +import com.redhat.satellite.search.config.Configuration; + +import org.picocontainer.defaults.DefaultPicoContainer; + +import junit.framework.TestCase; + +public abstract class BaseTestCase extends TestCase { + + protected DefaultPicoContainer container; + + @Override + protected void setUp() throws Exception { + System.getProperties().put("isTesting", "true"); + super.setUp(); + container = TestUtil.buildContainer(getComponentClasses()); + container.start(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + Configuration config = (Configuration) + container.getComponentInstanceOfType(Configuration.class); + TestUtil.cleanupDirectories(config); + container.stop(); + } + + @SuppressWarnings("unchecked") + protected abstract Class[] getComponentClasses(); + +} diff --git a/search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/TestUtil.java b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/TestUtil.java new file mode 100644 index 0000000..b86d80f --- /dev/null +++ b/search-server/spacewalk-search/src/java/com/redhat/satellite/search/tests/TestUtil.java @@ -0,0 +1,133 @@ +package com.redhat.satellite.search.tests; + +import com.redhat.satellite.search.config.Configuration; + +import org.picocontainer.defaults.DefaultPicoContainer; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.StringReader; +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class TestUtil { + + private TestUtil() { + + } + + @SuppressWarnings("unchecked") + public static DefaultPicoContainer buildContainer(Class[] components) throws IOException { + Configuration config = makeConfiguration(); + DefaultPicoContainer container = new DefaultPicoContainer(); + container.registerComponentInstance("appConfig", config); + for (int x = 0; x < components.length; x++) { + container.registerComponentImplementation(components[x]); + } + return container; + } + + @SuppressWarnings("unchecked") + public static Class[] buildComponentsList(Class component) { + Class[] arg = {component}; + return buildComponentsList(arg); + } + + @SuppressWarnings("unchecked") + public static Class[] buildComponentsList(Class[] components) { + List<Class>accum = new ArrayList<Class>(); + for (int x = 0; x < components.length; x++) { + processComponent(components[x], accum); + } + Class[] retval = new Class[accum.size()]; + accum.toArray(retval); + return retval; + } + + public static Configuration makeConfiguration() throws IOException { + String prefix = "search-server-test"; + String indexWorkDir = generateTempDir(prefix); + StringBuilder builder = new StringBuilder(); + builder.append("search.index_work_dir=").append(indexWorkDir).append("\n"); + builder.append("search.rpc_port=").append(generatePortNumber()).append("\n"); + builder.append("search.rpc_handlers="); + builder.append("index:com.redhat.satellite.search.rpc.handlers.IndexHandler\n\n"); + + // see if we overrode any properties in test.properties + File f = new File("src/config/test.properties"); + if (f.exists()) { + FileReader fr = new FileReader(f); + BufferedReader br = new BufferedReader(fr); + String line = null; + while ((line = br.readLine()) != null) { + builder.append(line); + builder.append("\n"); + } + } + + StringReader sr = new StringReader(builder.toString()); + return new Configuration(new BufferedReader(sr)); + } + + public static void cleanupDirectories(Configuration config) + throws IOException { + String workDir = config.getString("search.index_work_dir", null); + if (workDir != null) { + File f = new File(workDir); + if (f.exists() && f.isDirectory()) { + Runtime.getRuntime().exec("rm -rf " + workDir); + } + } + } + @SuppressWarnings("unchecked") + private static List<Class> processComponent(Class component, List<Class>accum) { + if (component == Configuration.class) { + return accum; + } + if (accum.indexOf(component) == -1) { + accum.add(component); + } + Constructor[] constructors = component.getConstructors(); + if (constructors != null) { + for (int x = 0;x < constructors.length; x++) { + Class[] paramTypes = constructors[x].getParameterTypes(); + if (paramTypes.length > 0) { + for (int y = 0;y < paramTypes.length; y++) { + if (isCandidate(paramTypes[y].getName())) { + processComponent(paramTypes[y], accum); + } + } + } + } + } + return accum; + } + + private static boolean isCandidate(String className) { + boolean retval = true; + if (className.startsWith("java.") || className.startsWith("javax.") || + className.startsWith("com.sun.")) { + retval = false; + } + return retval; + } + + private static String generateTempDir(String prefix) throws IOException { + File f = File.createTempFile(prefix, null); + f.delete(); + return f.getAbsolutePath(); + } + + private static String generatePortNumber() { + Random r = new Random(System.nanoTime()); + int port = 0; + while (port < 1024) { + port = r.nextInt(20001); + } + return String.valueOf(port); + } +} diff --git a/search-server/spacewalk-search/task-lib/ant-contrib-1.0.jar b/search-server/spacewalk-search/task-lib/ant-contrib-1.0.jar new file mode 100644 index 0000000..50b884f Binary files /dev/null and b/search-server/spacewalk-search/task-lib/ant-contrib-1.0.jar differ diff --git a/search-server/spacewalk-search/task-lib/antlr-2.7.6.jar b/search-server/spacewalk-search/task-lib/antlr-2.7.6.jar new file mode 100644 index 0000000..6f001d5 Binary files /dev/null and b/search-server/spacewalk-search/task-lib/antlr-2.7.6.jar differ diff --git a/search-server/spacewalk-search/task-lib/checkstyle-4.1.jar b/search-server/spacewalk-search/task-lib/checkstyle-4.1.jar new file mode 100644 index 0000000..49b639e Binary files /dev/null and b/search-server/spacewalk-search/task-lib/checkstyle-4.1.jar differ diff --git a/search-server/spacewalk-search/task-lib/commons-beanutils-1.7.0.jar b/search-server/spacewalk-search/task-lib/commons-beanutils-1.7.0.jar new file mode 100644 index 0000000..e024ab1 Binary files /dev/null and b/search-server/spacewalk-search/task-lib/commons-beanutils-1.7.0.jar differ diff --git a/search-server/spacewalk-search/task-lib/commons-logging-1.0.4.jar b/search-server/spacewalk-search/task-lib/commons-logging-1.0.4.jar new file mode 100644 index 0000000..e2f8cee Binary files /dev/null and b/search-server/spacewalk-search/task-lib/commons-logging-1.0.4.jar differ diff --git a/search-server/spacewalk-search/task-lib/emma-20040713.jar b/search-server/spacewalk-search/task-lib/emma-20040713.jar new file mode 100644 index 0000000..13e1118 Binary files /dev/null and b/search-server/spacewalk-search/task-lib/emma-20040713.jar differ diff --git a/search-server/spacewalk-search/task-lib/emma_ant-20040713.jar b/search-server/spacewalk-search/task-lib/emma_ant-20040713.jar new file mode 100644 index 0000000..76d47b8 Binary files /dev/null and b/search-server/spacewalk-search/task-lib/emma_ant-20040713.jar differ diff --git a/search-server/spacewalk-search/task-lib/ivy.jar b/search-server/spacewalk-search/task-lib/ivy.jar new file mode 100644 index 0000000..9421724 Binary files /dev/null and b/search-server/spacewalk-search/task-lib/ivy.jar differ diff --git a/search-server/spacewalk-search/task-lib/junit-3.8.1.jar b/search-server/spacewalk-search/task-lib/junit-3.8.1.jar new file mode 100644 index 0000000..e2e4783 Binary files /dev/null and b/search-server/spacewalk-search/task-lib/junit-3.8.1.jar differ diff --git a/search-server/spacewalk-search/task-lib/regexp-1.3.jar b/search-server/spacewalk-search/task-lib/regexp-1.3.jar new file mode 100644 index 0000000..8c96d98 Binary files /dev/null and b/search-server/spacewalk-search/task-lib/regexp-1.3.jar differ diff --git a/search-server/src/config/com/redhat/satellite/search/db/config.xml b/search-server/src/config/com/redhat/satellite/search/db/config.xml deleted file mode 100644 index a41e673..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/config.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMapConfig - PUBLIC "-//ibatis.apache.org//DTD SQL Map Config 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-config-2.dtd"> -<!-- Always ensure to use the correct XML header as above! --> -<sqlMapConfig> - <!-- These settings control SqlMap configuration details, primarily to do with transaction - management. They are all optional (see the Developer Guide for more). --> - <settings - cacheModelsEnabled="true" - statementCachingEnabled="false" - enhancementEnabled="true" - lazyLoadingEnabled="true" - maxRequests="100" - maxSessions="10" - maxTransactions="10" - useStatementNamespaces="false" - defaultStatementTimeout="30" - /> - <!-- Configure a datasource to use with this SQL Map using SimpleDataSource. - Notice the use of the properties from the above resource --> - <transactionManager type="JDBC" > - <dataSource type="SIMPLE"> - <property name="JDBC.Driver" value="${search.connection.driver_class}"/> - <property name="JDBC.ConnectionURL" value="${search.connection.url}"/> - <property name="JDBC.Username" value="${search.connection.username}"/> - <property name="JDBC.Password" value="${search.connection.password}"/> - </dataSource> - </transactionManager> - <!-- Identify all SQL Map XML files to be loaded by this SQL map. Notice the paths - are relative to the classpath. For now, we only have one
--> - <sqlMap resource="com/redhat/satellite/search/db/package.xml" /> - <sqlMap resource="com/redhat/satellite/search/db/errata.xml" /> - <sqlMap resource="com/redhat/satellite/search/db/server.xml" /> - <sqlMap resource="com/redhat/satellite/search/db/errata_handler.xml" /> - <sqlMap resource="com/redhat/satellite/search/db/hwdevice.xml" /> - <sqlMap resource="com/redhat/satellite/search/db/snapshot_tag.xml" /> - <sqlMap resource="com/redhat/satellite/search/db/server_custom_info.xml" /> -</sqlMapConfig> diff --git a/search-server/src/config/com/redhat/satellite/search/db/errata.xml b/search-server/src/config/com/redhat/satellite/search/db/errata.xml deleted file mode 100644 index 13c5985..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/errata.xml +++ /dev/null @@ -1,67 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMap - PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-2.dtd"> -<sqlMap namespace="errata"> - <select id="listErrataFromId" - parameterClass="long" - resultClass="com.redhat.satellite.search.db.models.Errata"> - SELECT - e.id, - e.advisory, - e.advisory_type as advisoryType, - e.advisory_name as advisoryName, - e.advisory_rel as advisoryRel, - e.product, - e.description, - e.synopsis, - e.topic, - e.solution, - e.issue_date as issueDate, - e.update_date as updateDate, - e.notes, - e.org_id, - e.created, - e.modified, - e.last_modified as lastModified - FROM rhnErrata e - WHERE e.id > #id# - ORDER by e.id ASC - </select> - <delete id="deleteLastErrata"> - DELETE rhnIndexerWork where object_type = 'errata' - </delete> - <insert id="createLastErrata" - parameterClass="java.util.Map"> - INSERT INTO rhnIndexerWork values ('errata', #id#, #last_modified#) - </insert> - <update id="updateLastErrata" - parameterClass="java.util.Map"> - UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'errata' - </update> - <select id="getLastErrataId" - resultClass="java.lang.Long"> - SELECT last_id from rhnIndexerWork where object_type='errata' - </select> - <select id="maxErrataId" resultClass="java.lang.Long"> - SELECT MAX(ID) - FROM rhnErrata - </select> - <select id="verifyErrataVisibility" resultClass="string"> - select distinct e.id - from rhnErrata e, - rhnChannelErrata ce, - rhnAvailableChannels ac, - pxrsessions ps, - web_contact wc - where e.id = ce.errata_id and - ce.channel_id = ac.channel_id and - ac.org_id = wc.org_id and - ps.web_user_id = wc.id and - ps.id = #sessionId# - and e.id IN - <iterate property="id_list" open="(" close=")" conjunction=","> - #id_list[]# - </iterate> - </select> -</sqlMap> diff --git a/search-server/src/config/com/redhat/satellite/search/db/errata_handler.xml b/search-server/src/config/com/redhat/satellite/search/db/errata_handler.xml deleted file mode 100644 index 7c6cece..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/errata_handler.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMap - PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-2.dtd"> -<sqlMap namespace="errataHandler"> - - <select id="listErrataByCVE" - parameterClass="Map" - resultClass="com.redhat.satellite.search.index.Result"> - SELECT DISTINCT - ecve.errata_id as id - from rhnErrataCVE ecve, - rhnCVE cve, - rhnErrata e, - rhnChannelErrata ce, - rhnAvailableChannels ac, - pxtsessions ps, - web_contact wc - where cve.id = ecve.cve_id and - e.id = ce.errata_id and - e.id = ecve.errata_id and - ce.channel_id = ac.channel_id and - ac.org_id = wc.org_id and - ps.web_user_id = wc.id and - ps.id = #sessionId# and - cve.name = #param0# - </select> - - <select id="listErrataByIssueDateRange" - parameterClass="Map" - resultClass="com.redhat.satellite.search.index.Result" > - SELECT - E.id as id - FROM rhnErrata E, - rhnChannelErrata ce, - rhnAvailableChannels ac, - pxtsessions ps, - web_contact wc - WHERE - e.id = ce.errata_id and - ce.channel_id = ac.channel_id and - ac.org_id = wc.org_id and - ps.web_user_id = wc.id and - ps.id = #sessionId# and - E.issue_date >= TO_DATE(#param0#, 'YYYY-MM-DD HH24:MI:SS') - AND E.issue_date <= TO_DATE(#param1#, 'YYYY-MM-DD HH24:MI:SS') - ORDER BY E.issue_date ASC - </select> -</sqlMap> diff --git a/search-server/src/config/com/redhat/satellite/search/db/hwdevice.xml b/search-server/src/config/com/redhat/satellite/search/db/hwdevice.xml deleted file mode 100644 index 70d6d8a..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/hwdevice.xml +++ /dev/null @@ -1,53 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMap - PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-2.dtd"> - -<sqlMap namespace="hwdevice"> - - <select id="getHardwareDeviceById" parameterClass="java.util.Map" - resultClass="com.redhat.satellite.search.db.models.HardwareDevice"> - SELECT - rd.id as id, - rd.server_id as serverId, - rd.class as classInfo, - rd.bus as bus, - rd.detached as detached, - rd.device as device, - rd.driver as driver, - rd.description as description, - rd.prop1 as vendorId, - rd.prop2 as deviceId, - rd.prop3 as subVendorId, - rd.prop4 as subDeviceId, - rd.pcitype as pciType - FROM rhnDevice rd - WHERE rd.id > #id# - ORDER by rd.id ASC - </select> - - - <delete id="deleteLastHardwareDevice"> - DELETE rhnIndexerWork where object_type = 'hwdevice' - </delete> - <insert id="createLastHardwareDevice" - parameterClass="java.util.Map"> - INSERT INTO rhnIndexerWork values ('hwdevice', #id#, #last_modified#) - </insert> - <update id="updateLastHardwareDevice" - parameterClass="java.util.Map"> - UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'hwdevice' - </update> - <select id="getLastHardwareDeviceId" - resultClass="java.lang.Long"> - SELECT last_id from rhnIndexerWork where object_type='hwdevice' - </select> - <select id="getLastHardwareDeviceIndexRun" - resultClass="java.util.Date"> - SELECT last_modified from rhnIndexerWork where object_type='hwdevice' - </select> - <select id="queryAllHwDeviceIds" resultClass="Long"> - SELECT id from rhnDevice - </select> - -</sqlMap> diff --git a/search-server/src/config/com/redhat/satellite/search/db/package.xml b/search-server/src/config/com/redhat/satellite/search/db/package.xml deleted file mode 100644 index 46e84d5..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/package.xml +++ /dev/null @@ -1,85 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMap - PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-2.dtd"> -<sqlMap namespace="package"> - <select id="getPackageById" - parameterClass="long" - resultClass="com.redhat.satellite.search.db.models.RhnPackage"> - SELECT - p.id as id, - pn.name as name, - pe.epoch as epoch, - pe.version as version, - pe.release as release, - pa.label as arch, - p.description as description, - p.summary as summary - FROM rhnPackage p, - rhnPackageName pn, - rhnPackageEVR pe, - rhnPackageArch pa - WHERE p.evr_id = pe.id - and p.name_id = pn.id - and p.package_arch_id = pa.id - and p.id = #id# - </select> - <select id="listPackagesFromId" - parameterClass="long" - resultClass="com.redhat.satellite.search.db.models.RhnPackage"> - SELECT - p.id as id, - pn.name as name, - pe.epoch as epoch, - pe.version as version, - pe.release as release, - pa.label as arch, - p.description as description, - p.summary as summary - FROM rhnPackage p, - rhnPackageName pn, - rhnPackageEVR pe, - rhnPackageArch pa - WHERE p.evr_id = pe.id - and p.name_id = pn.id - and p.package_arch_id = pa.id - and p.id > #id# - ORDER by p.id ASC - </select> - <delete id="deleteLastPackage"> - DELETE rhnIndexerWork where object_type = 'package' - </delete> - <insert id="createLastPackage" - parameterClass="java.util.Map"> - INSERT INTO rhnIndexerWork values ('package', #id#, #last_modified#) - </insert> - <update id="updateLastPackage" - parameterClass="java.util.Map"> - UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'package' - </update> - <select id="getLastPackageId" - resultClass="java.lang.Long"> - SELECT last_id from rhnIndexerWork where object_type='package' - </select> - <select id="maxPackageId" resultClass="java.lang.Long"> - SELECT MAX(ID) - FROM rhnPackage - </select> - <select id="verifyPackageVisibility" resultClass="string"> - select distinct p.id - from rhnPackage p, - rhnAvailableChannels ac, - rhnChannelPackage cp, - pxtsessions ps, - web_contact wc - where cp.package_id = p.id - and ac.channel_id = cp.channel_id - and ac.org_id = wc.org_id - and ps.web_user_id = wc.id - and ps.id = #session_id# - and p.id IN - <iterate property="id_list" open="(" close=")" conjunction=","> - #id_list[]# - </iterate> - </select> -</sqlMap> diff --git a/search-server/src/config/com/redhat/satellite/search/db/server.xml b/search-server/src/config/com/redhat/satellite/search/db/server.xml deleted file mode 100644 index 03f8b13..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/server.xml +++ /dev/null @@ -1,105 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMap - PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-2.dtd"> -<sqlMap namespace="server"> - <select id="getServerByIdOrDate" parameterClass="java.util.Map" - resultClass="com.redhat.satellite.search.db.models.Server"> - SELECT - s.id as id, - s.name as name, - s.description as description, - s.info as info, - s.created as registered, - s.running_kernel as runningKernel, - sl.machine as machine, - sl.rack as rack, - sl.room as room, - sl.building as building, - sl.address1 as address1, - sl.address2 as address2, - sl.city as city, - sl.state as state, - sl.country as country, - rsn.hostname as hostname, - rsn.ipaddr as ipaddr, - dmi.vendor as dmiVendor, - dmi.system as dmiSystem, - dmi.product as dmiProduct, - dmi.bios_vendor as dmiBiosVendor, - dmi.bios_version as dmiBiosVersion, - dmi.bios_release as dmiBiosRelease, - dmi.asset as dmiAsset, - dmi.board as dmiBoard, - cpu.bogomips as cpuBogoMIPS, - cpu.cache as cpuCache, - cpu.family as cpuFamily, - cpu.mhz as cpuMhz, - cpu.stepping as cpuStepping, - cpu.flags as cpuFlags, - cpu.model as cpuModel, - cpu.version as cpuVersion, - cpu.vendor as cpuVendor, - cpu.nrcpu as cpuNumberOfCpus, - cpu.acpiversion as cpuAcpiVersion, - cpu.apic as cpuApic, - cpu.apmversion as cpuApmVersion, - cpu.chipset as cpuChipset, - si.checkin as checkin, - rr.ram as ram, - rr.swap as swap - FROM rhnServer s - LEFT OUTER JOIN rhnServerNetwork rsn ON s.id = rsn.server_id - LEFT OUTER JOIN rhnServerInfo si on s.id = si.server_id - LEFT OUTER JOIN rhnRam rr on s.id = rr.server_id - LEFT OUTER JOIN rhnServerLocation sl ON s.id = sl.server_id - LEFT OUTER JOIN rhnServerDmi dmi ON s.id = dmi.server_id - LEFT OUTER JOIN rhnCPU cpu on s.id = cpu.server_id - WHERE s.modified >= #last_modified# - or rsn.modified >= #last_modified# - or si.checkin >= #last_modified# - or rr.modified >= #last_modified# - or sl.modified >= #last_modified# - or dmi.modified >= #last_modified# - or cpu.modified >= #last_modified# - or s.id > #id# - ORDER by s.id ASC - </select> - - <delete id="deleteLastServer"> - DELETE rhnIndexerWork where object_type = 'server' - </delete> - <insert id="createLastServer" - parameterClass="java.util.Map"> - INSERT INTO rhnIndexerWork values ('server', #id#, #last_modified#) - </insert> - <update id="updateLastServer" - parameterClass="java.util.Map"> - UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'server' - </update> - <select id="getLastServerId" - resultClass="java.lang.Long"> - SELECT last_id from rhnIndexerWork where object_type='server' - </select> - <select id="getLastServerIndexRun" - resultClass="java.util.Date"> - SELECT last_modified from rhnIndexerWork where object_type='server' - </select> - <select id="verifyServerVisibility" resultClass="string"> - SELECT object_id - FROM rhnVisibleObjects - WHERE pxt_session_id = #session_id# - AND object_type = #object_type# - AND object_id IN - <iterate property="id_list" open="(" close=")" conjunction=","> - #id_list[]# - </iterate> - </select> - <select id="maxServerId" resultClass="java.lang.Long"> - SELECT MAX(ID) - FROM rhnServer - </select> - <select id="queryAllServerIds" resultClass="Long"> - SELECT id from rhnServer - </select> -</sqlMap> diff --git a/search-server/src/config/com/redhat/satellite/search/db/server_custom_info.xml b/search-server/src/config/com/redhat/satellite/search/db/server_custom_info.xml deleted file mode 100644 index c56090c..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/server_custom_info.xml +++ /dev/null @@ -1,48 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMap - PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-2.dtd"> - -<sqlMap namespace="serverCustomInfo"> - - <select id="getServerCustomInfoByIdOrDate" parameterClass="java.util.Map" - resultClass="com.redhat.satellite.search.db.models.ServerCustomInfo"> - - SELECT - rscdv.key_id as keyId, - rscdv.server_id as serverId, - rscdv.value as value, - rscdv.created_by as createdBy, - rscdv.last_modified_by as lastModifiedBy, - rcdk.label as label, - TO_CHAR(rscdv.created, 'YYYY-MM-DD HH24-MI-SS') as created, - TO_CHAR(rscdv.modified, 'YYYY-MM-DD HH24-MI-SS') as modified - FROM rhnServerCustomDataValue rscdv - INNER JOIN rhnCustomDataKey rcdk on rcdk.id = rscdv.key_id - WHERE rscdv.modified > #last_modified# - ORDER by rscdv.key_id ASC - </select> - - <delete id="deleteLastServerCustomInfo"> - DELETE rhnIndexerWork where object_type = 'server_custom_info' - </delete> - <insert id="createLastServerCustomInfo" - parameterClass="java.util.Map"> - INSERT INTO rhnIndexerWork values ('server_custom_info', #id#, #last_modified#) - </insert> - <update id="updateLastServerCustomInfo" - parameterClass="java.util.Map"> - UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'server_custom_info' - </update> - <select id="getLastServerCustomInfoId" - resultClass="java.lang.Long"> - SELECT last_id from rhnIndexerWork where object_type='server_custom_info' - </select> - <select id="getLastServerCustomInfoIndexRun" - resultClass="java.util.Date"> - SELECT last_modified from rhnIndexerWork where object_type='server_custom_info' - </select> - <select id="queryAllServerCustomInfoIds" resultClass="com.redhat.satellite.search.db.models.ServerCustomInfo"> - SELECT server_id as serverId, key_id as keyId from rhnServerCustomDataValue - </select> -</sqlMap> diff --git a/search-server/src/config/com/redhat/satellite/search/db/snapshot_tag.xml b/search-server/src/config/com/redhat/satellite/search/db/snapshot_tag.xml deleted file mode 100644 index 5d4d67c..0000000 --- a/search-server/src/config/com/redhat/satellite/search/db/snapshot_tag.xml +++ /dev/null @@ -1,53 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!DOCTYPE sqlMap - PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" - "http://ibatis.apache.org/dtd/sql-map-2.dtd"> - -<sqlMap namespace="snapshotTag"> - - <select id="getSnapshotTagByIdOrDate" parameterClass="java.util.Map" - resultClass="com.redhat.satellite.search.db.models.SnapshotTag"> - - SELECT - rt.id as id, - rst.snapshot_id as snapshotId, - rtn.id as tagNameId, - rst.server_id as serverId, - rt.org_id as orgId, - rtn.name as name, - TO_CHAR(rst.created, 'YYYY-MM-DD HH24-MI-SS') as created, - TO_CHAR(rst.modified, 'YYYY-MM-DD HH24-MI-SS') as modified - FROM rhnSnapshotTag rst - INNER JOIN rhnTag rt on rst.tag_id = rt.id - INNER JOIN rhnTagName rtn on rt.name_id = rtn.id - WHERE rst.tag_id > #id# OR rst.modified > #last_modified# - ORDER by rst.snapshot_id ASC - </select> - - - <delete id="deleteLastSnapshotTag"> - DELETE rhnIndexerWork where object_type = 'snapshot_tag' - </delete> - <insert id="createLastSnapshotTag" - parameterClass="java.util.Map"> - INSERT INTO rhnIndexerWork values ('snapshot_tag', #id#, #last_modified#) - </insert> - <update id="updateLastSnapshotTag" - parameterClass="java.util.Map"> - UPDATE rhnIndexerWork SET last_id = #id#, last_modified = #last_modified# where object_type = 'snapshot_tag' - </update> - <select id="getLastSnapshotTagId" - resultClass="java.lang.Long"> - SELECT last_id from rhnIndexerWork where object_type='snapshot_tag' - </select> - <select id="getLastSnapshotTagIndexRun" - resultClass="java.util.Date"> - SELECT last_modified from rhnIndexerWork where object_type='snapshot_tag' - </select> - - <select id="queryAllSnapshotTagIds" resultClass="Long"> - SELECT snapshot_id from rhnSnapshotTag - </select> - - -</sqlMap> diff --git a/search-server/src/config/etc/logrotate.d/rhn-search b/search-server/src/config/etc/logrotate.d/rhn-search deleted file mode 100644 index a2793cb..0000000 --- a/search-server/src/config/etc/logrotate.d/rhn-search +++ /dev/null @@ -1,9 +0,0 @@ -/var/log/rhn/search/rhn_search.log { - weekly - rotate 5 - copytruncate - compress - notifempty - missingok - size=10M -} diff --git a/search-server/src/config/log4j.properties b/search-server/src/config/log4j.properties deleted file mode 100644 index 6c2195a..0000000 --- a/search-server/src/config/log4j.properties +++ /dev/null @@ -1,35 +0,0 @@ -log4j.appender.RootAppender=org.apache.log4j.ConsoleAppender -log4j.appender.RootAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.RootAppender.layout.ConversionPattern=%d [%t] %-5p %c - %m%n -log4j.rootLogger=WARN,RootAppender - -## XmlRpc Logging -log4j.appender.SearchAppender=org.apache.log4j.FileAppender -log4j.appender.SearchAppender.file=/var/log/rhn/search/rhn_search.log -log4j.appender.SearchAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.SearchAppender.layout.ConversionPattern=[%d] %-5p - %m%n -log4j.logger.com.redhat.satellite=WARN,SearchAppender -log4j.logger.org.quartz=WARN,SearchAppender -#log4j.logger.net.rkbloom.logdriver.LogPreparedStatement=DEBUG -#log4j.logger.net.rkbloom.logdriver.LogStatement=DEBUG -#log4j.logger.net.rkbloom.logdriver.LogCallableStatement=DEBUG -#log4j.logger.net.rkbloom.logdriver.LogConnection=DEBUG -#log4j.logger.com.redhat.satellite.search.rpc.handlers=DEBUG,SearchAppender -#log4j.logger.com.redhat.satellite.search.index.IndexManager=DEBUG,SearchAppender -#log4j.logger.com.redhat.satellite.search.ngram.NGramQueryParser=DEBUG,SearchAppender -#log4j.logger.com.redhat.satellite.search.scheduler.tasks=DEBUG,SearchAppender -#log4j.logger.com.redhat.satellite.search.rpc.handlers=DEBUG,SearchAppender - -# SqlMap logging configuration... -#log4j.logger.com.ibatis=DEBUG -#log4j.logger.com.ibatis.common.jdbc.SimpleDataSource=DEBUG -#log4j.logger.com.ibatis.sqlmap.engine.cache.CacheModel=DEBUG -#log4j.logger.com.ibatis.sqlmap.engine.impl.SqlMapClientImpl=DEBUG -#log4j.logger.com.ibatis.sqlmap.engine.builder.xml.SqlMapParser=DEBUG -#log4j.logger.com.ibatis.common.util.StopWatch=DEBUG -#log4j.logger.java.sql.Connection=DEBUG -#log4j.logger.java.sql.Statement=DEBUG -#log4j.logger.java.sql.PreparedStatement=DEBUG -#log4j.logger.java.sql.ResultSet=DEBUG - - diff --git a/search-server/src/config/log4j.test.properties b/search-server/src/config/log4j.test.properties deleted file mode 100644 index 72e7ac6..0000000 --- a/search-server/src/config/log4j.test.properties +++ /dev/null @@ -1,17 +0,0 @@ -log4j.appender.RootAppender=org.apache.log4j.ConsoleAppender -log4j.appender.RootAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.RootAppender.layout.ConversionPattern=%d [%t] %-5p %c - %m%n -log4j.appender.RootAppender.Threshold=WARN -log4j.rootLogger=INFO,RootAppender - -log4j.appender.SearchAppender=org.apache.log4j.FileAppender -log4j.appender.SearchAppender.file=test_rhn_search.log -log4j.appender.SearchAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.SearchAppender.layout.ConversionPattern=[%d] %-5p - %m%n -log4j.appender.SearchAppender.Threshold=DEBUG - -log4j.logger.com.redhat.satellite=INFO,SearchAppender -log4j.logger.com.redhat.satellite.search.config=INFO - -log4j.logger.java.sql.ResultSet=WARN -log4j.logger.com.redhat.satellite.search.index.ngram.tests=DEBUG,RootAppender diff --git a/search-server/src/config/rhn-search b/search-server/src/config/rhn-search deleted file mode 100644 index f7856b8..0000000 --- a/search-server/src/config/rhn-search +++ /dev/null @@ -1,395 +0,0 @@ -#! /bin/sh -# -# rhn-search This shell script takes care of starting and stopping -# the java rhn-search server. -# -# chkconfig: - 94 12 -# description: This is Spacewalk search engine. -# -### BEGIN INIT INFO -# Provides: rhn-search -# Required-Start: $local_fs $network satellite-httpd -# Required-Stop: $local_fs $network satellite-httpd -# Default-Start: -# Default-Stop: 0 1 2 3 4 5 6 -# Short-Description: This is Spacewalk search engine. -# Description: This is Spacewalk search engine. -### END INIT INFO - -# -# Skeleton sh script suitable for starting and stopping -# wrapped Java apps on the Solaris platform. -# -# Make sure that PIDFILE points to the correct location, -# if you have changed the default location set in the -# wrapper configuration file. -# - -#----------------------------------------------------------------------------- -# These settings can be modified to fit the needs of your application - -# Application -APP_NAME="rhn-search" -APP_LONG_NAME="rhn-search" - -# Wrapper -WRAPPER_CMD="/usr/bin/rhnsearchd" -WRAPPER_CONF="/etc/rhn/search/rhn_search_daemon.conf" - -# Priority at which to run the wrapper. See "man nice" for valid priorities. -# nice is only used if a priority is specified. -PRIORITY= - -# Location of the pid file. -PIDDIR="/var/run" - -# If uncommented, causes the Wrapper to be shutdown using an anchor file. -# When launched with the 'start' command, it will also ignore all INT and -# TERM signals. -#IGNORE_SIGNALS=true - -# If specified, the Wrapper will be run as the specified user when the 'start' -# command is passed to this script. When running with the 'console' command -# the current user will be used. -# IMPORTANT - Make sure that the user has the required privileges to write -# the PID file and wrapper.log files. Failure to be able to write the log -# file will cause the Wrapper to exit without any way to write out an error -# message. -# NOTE - This will set the user which is used to run the Wrapper as well as -# the JVM and is not useful in situations where a privileged resource or -# port needs to be allocated prior to the user being changed. -#RUN_AS_USER= - -unlock() { - IDXDIR=`grep search.index_work_dir /etc/rhn/rhn.conf` - if [ -z $IDXDIR ] - then - IDXDIR=`grep search.index_work_dir /etc/rhn/search/rhn_search.conf` - fi - - if [ ! -z $IDXDIR ] - then - IDXDIR=`echo $IDXDIR | sed -e 's/=/ /g' | awk '{print $2}'` - find $IDXDIR -type f -name write.lock -exec rm -f {} ; - fi -} - -removeindex() { - SEARCH_CLASS_PATH=`build-classpath-directory /usr/share/rhn/search/lib /usr/share/java/ojdbc14.jar` - CLASSPATH=${SEARCH_CLASS_PATH} java com.redhat.satellite.search.DeleteIndexes -} - -# Do not modify anything beyond this point -#----------------------------------------------------------------------------- - -# Get the fully qualified path to the script -case $0 in - /*) - SCRIPT="$0" - ;; - *) - PWD=`pwd` - SCRIPT="$PWD/$0" - ;; -esac - -# Change spaces to ":" so the tokens can be parsed. -SCRIPT=`echo $SCRIPT | sed -e 's; ;:;g'` -# Get the real path to this script, resolving any symbolic links -TOKENS=`echo $SCRIPT | sed -e 's;/; ;g'` -REALPATH= -for C in $TOKENS; do - REALPATH="$REALPATH/$C" - while [ -h "$REALPATH" ] ; do - LS="`ls -ld "$REALPATH"`" - LINK="`expr "$LS" : '.*-> (.*)$'`" - if expr "$LINK" : '/.*' > /dev/null; then - REALPATH="$LINK" - else - REALPATH="`dirname "$REALPATH"`""/$LINK" - fi - done -done -# Change ":" chars back to spaces. -REALPATH=`echo $REALPATH | sed -e 's;:; ;g'` - -# Change the current directory to the location of the script -cd "`dirname "$REALPATH"`" - -# Process ID -ANCHORFILE="$PIDDIR/$APP_NAME.anchor" -PIDFILE="$PIDDIR/$APP_NAME.pid" -pid="" - -# Resolve the location of the 'ps' command -PSEXE="/usr/bin/ps" -if [ ! -x $PSEXE ] -then - PSEXE="/bin/ps" - if [ ! -x $PSEXE ] - then - echo "Unable to locate 'ps'." - echo "Please report this message along with the location of the command on your system." - exit 1 - fi -fi - -# Build the nice clause -if [ "X$PRIORITY" = "X" ] -then - CMDNICE="" -else - CMDNICE="nice -$PRIORITY" -fi - -# Check the configured user -if [ "X$RUN_AS_USER" != "X" ] -then - # Resolve the location of the 'id' command - IDEXE="/usr/xpg4/bin/id" - if [ ! -x $IDEXE ] - then - IDEXE="/usr/bin/id" - if [ ! -x $IDEXE ] - then - echo "Unable to locate 'id'." - echo "Please report this message along with the location of the command on your system." - exit 1 - fi - fi - - if [ "`$IDEXE -u -n`" = "$RUN_AS_USER" ] - then - # Already running as the configured user. Avoid password prompts by not calling runuser. - RUN_AS_USER="" - fi -fi - -getpid() { - if [ -f $PIDFILE ] - then - if [ -r $PIDFILE ] - then - pid=`cat $PIDFILE` - if [ "X$pid" != "X" ] - then - # Verify that a process with this pid is still running. - pid=`$PSEXE -p $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` - if [ "X$pid" = "X" ] - then - # This is a stale pid file. - rm -f $PIDFILE - echo "Removed stale pid file: $PIDFILE" - fi - fi - else - echo "Cannot read $PIDFILE." - exit 1 - fi - fi -} - -testpid() { - pid=`$PSEXE -p $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` - if [ "X$pid" = "X" ] - then - # Process is gone so remove the pid file. - rm -f $PIDFILE - fi -} - -console() { - echo "Running $APP_LONG_NAME..." - getpid - if [ "X$pid" = "X" ] - then - if [ "X$IGNORE_SIGNALS" = "X" ] - then - exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE - else - exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.anchorfile=$ANCHORFILE - fi - else - echo "$APP_LONG_NAME is already running." - exit 1 - fi -} - -start() { - echo "Starting $APP_LONG_NAME..." - getpid - if [ "X$pid" = "X" ] - then - if [ "X$IGNORE_SIGNALS" = "X" ] - then - if [ "X$RUN_AS_USER" = "X" ] - then - exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.daemonize=TRUE - else - /sbin/runuser -m $RUN_AS_USER -c "exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.daemonize=TRUE" - fi - else - if [ "X$RUN_AS_USER" = "X" ] - then - exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.anchorfile=$ANCHORFILE wrapper.ignore_signals=TRUE wrapper.daemonize=TRUE - else - /sbin/runuser -m $RUN_AS_USER -c "exec $CMDNICE $WRAPPER_CMD $WRAPPER_CONF wrapper.pidfile=$PIDFILE wrapper.anchorfile=$ANCHORFILE wrapper.ignore_signals=TRUE wrapper.daemonize=TRUE" - fi - fi - else - echo "$APP_LONG_NAME is already running." - exit 1 - fi -} - -stopit() { - echo "Stopping $APP_LONG_NAME..." - getpid - if [ "X$pid" = "X" ] - then - echo "$APP_LONG_NAME was not running." - else - if [ "X$IGNORE_SIGNALS" = "X" ] - then - # Running so try to stop it. - kill $pid - if [ $? -ne 0 ] - then - # An explanation for the failure should have been given - echo "Unable to stop $APP_LONG_NAME." - exit 1 - fi - else - rm -f $ANCHORFILE - if [ -f $ANCHORFILE ] - then - # An explanation for the failure should have been given - echo "Unable to stop $APP_LONG_NAME." - exit 1 - fi - fi - - # We can not predict how long it will take for the wrapper to - # actually stop as it depends on settings in wrapper.conf. - # Loop until it does. - savepid=$pid - CNT=0 - TOTCNT=0 - while [ "X$pid" != "X" ] - do - # Loop for up to 5 minutes - if [ "$TOTCNT" -lt "300" ] - then - if [ "$CNT" -lt "5" ] - then - CNT=`expr $CNT + 1` - else - echo "Waiting for $APP_LONG_NAME to exit..." - CNT=0 - fi - TOTCNT=`expr $TOTCNT + 1` - - sleep 1 - - testpid - else - pid= - fi - done - - pid=$savepid - testpid - if [ "X$pid" != "X" ] - then - echo "Timed out waiting for $APP_LONG_NAME to exit." - echo " Attempting a forced exit..." - kill -9 $pid - fi - - pid=$savepid - testpid - if [ "X$pid" != "X" ] - then - echo "Failed to stop $APP_LONG_NAME." - exit 1 - else - echo "Stopped $APP_LONG_NAME." - fi - fi -} - -status() { - getpid - if [ "X$pid" = "X" ] - then - echo "$APP_LONG_NAME is not running." - exit 1 - else - echo "$APP_LONG_NAME is running ($pid)." - exit 0 - fi -} - -dump() { - echo "Dumping $APP_LONG_NAME..." - getpid - if [ "X$pid" = "X" ] - then - echo "$APP_LONG_NAME was not running." - - else - kill -3 $pid - - if [ $? -ne 0 ] - then - echo "Failed to dump $APP_LONG_NAME." - exit 1 - else - echo "Dumped $APP_LONG_NAME." - fi - fi -} - -case "$1" in - - 'console') - console - ;; - - 'start') - start - ;; - - 'stop') - stopit - unlock - ;; - - 'restart') - stopit - unlock - start - ;; - - 'status') - status - ;; - - 'dump') - dump - ;; - - 'cleanindex') - stopit - unlock - removeindex - start - ;; - - *) - echo "Usage: $0 { console | start | stop | restart | status | dump | cleanindex }" - exit 1 - ;; -esac - -exit 0 diff --git a/search-server/src/config/search/rhn_search.conf b/search-server/src/config/search/rhn_search.conf deleted file mode 100644 index 6a3cb7d..0000000 --- a/search-server/src/config/search/rhn_search.conf +++ /dev/null @@ -1,15 +0,0 @@ -search.index_work_dir=/usr/share/rhn/search/indexes -search.rpc_handlers=index:com.redhat.satellite.search.rpc.handlers.IndexHandler,db:com.redhat.satellite.search.rpc.handlers.DatabaseHandler,admin:com.redhat.satellite.search.rpc.handlers.AdminHandler -search.max_hits_returned=500 -search.connection.driver_class=oracle.jdbc.driver.OracleDriver -search.score_threshold=.10 -search.system_score_threshold=.01 -search.errata_score_threshold=.20 -search.errata.advisory_score_threshold=.30 -search.min_ngram = 1 -search.max_ngram = 5 -search.doc.limit_results = false -search.schedule.interval = 300000 -search.log.explain.results = false - - diff --git a/search-server/src/config/search/rhn_search_daemon.conf b/search-server/src/config/search/rhn_search_daemon.conf deleted file mode 100644 index 07d7145..0000000 --- a/search-server/src/config/search/rhn_search_daemon.conf +++ /dev/null @@ -1,86 +0,0 @@ -#******************************************************************** -# Wrapper Properties -#******************************************************************** -# Java Application -wrapper.java.command=/usr/bin/java - -# Java Main class. This class must implement the WrapperListener interface -# or guarantee that the WrapperManager class is initialized. Helper -# classes are provided to do this for you. See the Integration section -# of the documentation for details. -wrapper.java.mainclass=com.redhat.satellite.search.Main - -# Java Classpath (include wrapper.jar) Add class path elements as -# needed starting from 1 -wrapper.java.classpath.1=/usr/share/rhn/search/classes/ -wrapper.java.classpath.2=/usr/share/rhn/search/lib/commons-lang.jar -wrapper.java.classpath.3=/usr/share/rhn/search/lib/commons-logging.jar -wrapper.java.classpath.4=/usr/share/rhn/search/lib/ibatis-2.3.0.677.jar -wrapper.java.classpath.5=/usr/share/rhn/search/lib/log4j.jar -wrapper.java.classpath.6=/usr/share/rhn/search/lib/lucene-core-2.3.2.jar -wrapper.java.classpath.7=/usr/share/java/ojdbc14.jar -wrapper.java.classpath.8=/usr/share/rhn/search/lib/picocontainer-1.3.jar -wrapper.java.classpath.9=/usr/share/rhn/search/lib/quartz.jar -wrapper.java.classpath.10=/usr/share/rhn/search/lib/redstone-xmlrpc.jar -wrapper.java.classpath.11=/usr/share/rhn/search/lib/redstone-xmlrpc-client.jar -wrapper.java.classpath.12=/usr/share/rhn/search/lib/simple-core-3.1.3.jar -wrapper.java.classpath.13=/usr/share/rhn/search/lib/spacewalk-search.jar -wrapper.java.classpath.14=/usr/share/java/tanukiwrapper.jar -wrapper.java.classpath.15=/usr/share/rhn/search/lib/lucene-analyzers-2.3.2.jar -wrapper.java.classpath.16=/usr/share/rhn/search/lib/hadoop-0.18.1-core.jar -# -#Remember we want nutch/conf to be found ahead of the nutch.jar -#this ensures the config files used are ours and not the default ones -#in the jar. -# -wrapper.java.classpath.17=/usr/share/nutch -wrapper.java.classpath.18=/usr/share/nutch/conf -wrapper.java.classpath.19=/usr/share/rhn/search/lib/nutch-2008-12-01_04-01-21.jar -wrapper.java.classpath.20=/usr/share/rhn/search/lib/commons-httpclient-3.0.jar - -# Java Library Path (location of Wrapper.DLL or libwrapper.so) -wrapper.java.library.path.1=/usr/lib -wrapper.java.library.path.2=/usr/lib64 - -# Java Additional Parameters -#wrapper.java.additional.1= - -# Initial Java Heap Size (in MB) -#wrapper.java.initmemory=32 - -# Maximum Java Heap Size (in MB) -#wrapper.java.maxmemory=96 - -# Application parameters. Add parameters as needed starting from 1 -#wrapper.app.parameter.1=<YourMainClass> - -#******************************************************************** -# Wrapper Logging Properties -#******************************************************************** -# Format of output for the console. (See docs for formats) -wrapper.console.format=PM - -# Log Level for console output. (See docs for log levels) -wrapper.console.loglevel=INFO - -# Log file to use for wrapper output logging. -wrapper.logfile=/var/log/rhn/search/rhn_search_daemon.log - -# Format of output for the log file. (See docs for formats) -wrapper.logfile.format=LPTM - -# Log Level for log file output. (See docs for log levels) -wrapper.logfile.loglevel=INFO - -# Maximum size that the log file will be allowed to grow to before -# the log is rolled. Size is specified in bytes. The default value -# of 0, disables log rolling. May abbreviate with the 'k' (kb) or -# 'm' (mb) suffix. For example: 10m = 10 megabytes. -wrapper.logfile.maxsize=5m - -# Maximum number of rolled log files which will be allowed before old -# files are deleted. The default value of 0 implies no limit. -wrapper.logfile.maxfiles=0 - -# Log Level for sys/event log output. (See docs for log levels) -wrapper.syslog.loglevel=NONE diff --git a/search-server/src/config/test.properties b/search-server/src/config/test.properties deleted file mode 100644 index a37fd98..0000000 --- a/search-server/src/config/test.properties +++ /dev/null @@ -1,5 +0,0 @@ -# test config file DO NOT SHIP -search.connection.username = spacewalk -search.connection.password = spacewalk -search.connection.url=jdbc:log:oracle.jdbc.driver.OracleDriver:oracle:thin:@localhost:1521:xe -search.connection.driver_class=net.rkbloom.logdriver.LogDriver diff --git a/search-server/src/java/com/redhat/satellite/search/ContainerRunner.java b/search-server/src/java/com/redhat/satellite/search/ContainerRunner.java deleted file mode 100644 index 65e089a..0000000 --- a/search-server/src/java/com/redhat/satellite/search/ContainerRunner.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search; - -import org.picocontainer.defaults.DefaultPicoContainer; - -/** - * Runs a PicoContainer instance in a separate thread - * to prevent the main process from exiting - * - * @version $Rev$ - */ -class ContainerRunner implements Runnable { - - private DefaultPicoContainer container; - - /** - * Constructor - * @param containerIn PicoContainer instance to use - */ - ContainerRunner(DefaultPicoContainer containerIn) { - container = containerIn; - } - - /** - * {@inheritDoc} - */ - public void run() { - container.start(); - try { - synchronized (this) { - this.wait(); - } - container.stop(); - } - catch (InterruptedException e) { - container.stop(); - } - } - - /** - * Stop the thread - */ - public void stop() { - synchronized (this) { - this.notify(); - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/DeleteIndexes.java b/search-server/src/java/com/redhat/satellite/search/DeleteIndexes.java deleted file mode 100644 index 81206ab..0000000 --- a/search-server/src/java/com/redhat/satellite/search/DeleteIndexes.java +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search; - -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.WriteQuery; -import com.redhat.satellite.search.config.Configuration; - - -/** - * Reindex - cleans up indexes on filesystem and database so reindexing will occur - * @version $Rev: 1 $ - */ -public class DeleteIndexes { - private static Logger log = Logger.getLogger(DeleteIndexes.class); - - private DeleteIndexes() { - } - - protected static boolean deleteDirectory(File dir) { - File[] files = dir.listFiles(); - boolean warning = true; - for (int i = 0; i < files.length; i++) { - if (files[i].isDirectory()) { - deleteDirectory(files[i]); - } - if (files[i].delete()) { - log.debug("Deleted: " + files[i].getAbsolutePath()); - } - else { - log.warn("*ERROR* unable to delete: " + files[i].getAbsolutePath()); - warning = false; - } - } - if (!dir.delete()) { - log.warn("*ERROR* unable to delete: " + dir.getAbsolutePath()); - warning = false; - } - return warning; - } - - protected static boolean deleteIndexPath(String path) { - File dir = new File(path); - if ("/".equals(dir.getAbsolutePath())) { - log.warn("Error, passed in path is <" + path + "> this looks wrong"); - return false; - } - if (!dir.exists()) { - log.debug("Path <" + dir.getAbsolutePath() + "> doesn't exist"); - return true; // dir doesn't exist, so just as good as deleted - } - if (!dir.isDirectory()) { - log.warn("Error, passed in path <" + path + "> is not a directory"); - return false; - } - log.info("Attempting to delete " + dir.getAbsolutePath()); - return deleteDirectory(dir); - } - - protected static void deleteQuery(DatabaseManager databaseManager, - String queryName) throws SQLException { - WriteQuery query = null; - log.info("Running query: " + queryName); - query = databaseManager.getWriterQuery(queryName); - query.delete(null); - query.close(); - } - - /** - * @param args - */ - public static void main(String[] args) { - try { - Configuration config = new Configuration(); - DatabaseManager databaseManager = new DatabaseManager(config); - String indexWorkDir = config.getString("search.index_work_dir", null); - if (StringUtils.isBlank(indexWorkDir)) { - log.warn("Couldn't find path for where index files are stored."); - log.warn("Looked in config for property: search.index_work_dir"); - return; - } - List<IndexInfo> indexes = new ArrayList<IndexInfo>(); - indexes.add(new IndexInfo("deleteLastErrata", - indexWorkDir + File.separator + "errata")); - indexes.add(new IndexInfo("deleteLastPackage", - indexWorkDir + File.separator + "package")); - indexes.add(new IndexInfo("deleteLastServer", - indexWorkDir + File.separator + "server")); - indexes.add(new IndexInfo("deleteLastHardwareDevice", - indexWorkDir + File.separator + "hwdevice")); - indexes.add(new IndexInfo("deleteLastSnapshotTag", - indexWorkDir + File.separator + "snapshotTag")); - indexes.add(new IndexInfo("deleteLastServerCustomInfo", - indexWorkDir + File.separator + "serverCustomInfo")); - for (IndexInfo info : indexes) { - deleteQuery(databaseManager, info.getQueryName()); - if (!deleteIndexPath(info.getDirPath())) { - log.warn("Failed to delete index for " + info.getDirPath()); - } - } - } - catch (SQLException e) { - log.error("Caught Exception: ", e); - if (e.getErrorCode() == 17002) { - log.error("Unable to establish database connection."); - log.error("Ensure database is available and connection details are " + - "correct, then retry"); - } - System.exit(1); - } - catch (IOException e) { - log.error("Caught Exception: ", e); - System.exit(1); - } - log.info("Index files have been deleted and database has been cleaned up, " + - "ready to reindex"); - } - - /** - * IndexInfo - */ - protected static class IndexInfo { - protected String queryName; - protected String dirPath; - - /** - * - * @param query query name to delete all records - * @param path string pointing to where index files reside - */ - public IndexInfo(String query, String path) { - queryName = query; - dirPath = path; - } - - /** - * Set the query name to delete all records for this index - * @param queryNameIn name of the query - */ - public void setQueryName(String queryNameIn) { - queryName = queryNameIn; - } - /** - * Returns the query name - * @return - */ - public String getQueryName() { - return queryName; - } - /** - * Set the path where the index files exist - * @param path String which points to index files - */ - public void setDirPath(String path) { - dirPath = path; - } - /** - * Returns the path - * @return a string which points to the index files - */ - public String getDirPath() { - return dirPath; - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/Main.java b/search-server/src/java/com/redhat/satellite/search/Main.java deleted file mode 100644 index 9ff0d9d..0000000 --- a/search-server/src/java/com/redhat/satellite/search/Main.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search; - -import com.redhat.satellite.search.config.Configuration; -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.rpc.RpcServer; -import com.redhat.satellite.search.scheduler.ScheduleManager; - -import org.apache.log4j.Logger; -import org.picocontainer.defaults.DefaultPicoContainer; -import org.tanukisoftware.wrapper.WrapperListener; -import org.tanukisoftware.wrapper.WrapperManager; - -/** - * Entry point for the Tanuki daemon wrapper - * - * @version $Rev$ - */ -public class Main implements WrapperListener { - - private static Logger log = Logger.getLogger(Main.class); - private static final Class[] COMPONENTS = {DatabaseManager.class, - IndexManager.class, - RpcServer.class, - ScheduleManager.class}; - - private DefaultPicoContainer container; - private ContainerRunner runner; - - /** - * {@inheritDoc} - */ - public void controlEvent(int arg) { - } - - /** - * {@inheritDoc} - */ - public Integer start(String[] argv) { - Configuration config = new Configuration(); - container = new DefaultPicoContainer(); - container.registerComponentInstance(config); - for (int x = 0; x < COMPONENTS.length; x++) { - container.registerComponentImplementation(COMPONENTS[x]); - } - runner = new ContainerRunner(container); - Thread t = new Thread(runner); - t.setDaemon(false); - t.start(); - - return null; - } - - /** - * {@inheritDoc} - */ - public int stop(int arg) { - log.info("Stopping Main"); - runner.stop(); - return 0; - } - - /** - * Main entry point - * @param argv command-line args - */ - public static void main(String[] argv) { - Main m = new Main(); - WrapperManager.start(m, argv); - if (log.isDebugEnabled()) { - log.debug("Returned from WrapperManager.start"); - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/config/ConfigException.java b/search-server/src/java/com/redhat/satellite/search/config/ConfigException.java deleted file mode 100644 index c83c7ea..0000000 --- a/search-server/src/java/com/redhat/satellite/search/config/ConfigException.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -/* - * AUTOMATICALLY GENERATED FILE, DO NOT EDIT. - */ -package com.redhat.satellite.search.config; - -/** - * Base class for all Config exceptions - * <p> - - * - * @version definition($Rev: 76724 $)/template($Rev: 67725 $) - */ -public class ConfigException extends RuntimeException { - - /** - * Constructor - * @param message exception message - */ - public ConfigException(String message) { - super(message); - } - - /** - * Constructor - * @param message exception message - * @param cause the cause (which is saved for later retrieval - * by the Throwable.getCause() method). (A null value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) - */ - public ConfigException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/config/Configuration.java b/search-server/src/java/com/redhat/satellite/search/config/Configuration.java deleted file mode 100644 index 1971b26..0000000 --- a/search-server/src/java/com/redhat/satellite/search/config/Configuration.java +++ /dev/null @@ -1,622 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.config; - -import com.redhat.satellite.search.config.translator.TranslatorRegistry; - -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.TreeSet; - -/** - * The Config class acts as an abstraction layer between our configuration - * system, and the actual implementation. The basic idea is that there is a - * global config, but you can instantiate one of your own if you want. This - * layer insulates us from changing the underlying implementation. - * <p> - * Config files are properties, with /etc/rhn/default/rhn.conf setting defaults - * that can be overridden /etc/rhn/rhn.conf. - * - * @version $Rev$ - */ -public class Configuration { - - private static Logger logger = Logger.getLogger(Configuration.class); - - // - // Location of config files - // - - /** - * The default directory in which to look for config files - */ - public static final String DEFAULT_CONF_DIR = "/etc/rhn"; - - /** - * The system property containing the RHN configuration directory. - * If the property is not set, config files are read - * from {@link #DEFAULT_CONF_DIR} - */ - private static final String RHN_CONF_DIR_PROPERTY = "rhn.config.dir"; - - /** - * The system property containing the SearchServer configuration directory. - * If the property is not set, config files are read - * from {@link #DEFAULT_CONF_DIR}/search - */ - private static final String SEARCH_CONF_DIR_PROPERTY = "search.config.dir"; - - - /** - * List of values that are considered true, ignoring case. - */ - private static final String[] TRUE_VALUES = {"1", "y", "true", "yes", "on"}; - - /** - * array of prefix in the order they should be search - * if the given lookup string is without a namespace. - */ - private String[] prefixOrder = new String[] {"web", "server"}; - - /** hash of configuration properties */ - private Properties configValues = new Properties(); - - private List<KeyTranslator> translators; - - /** set of configuration file names */ - private TreeSet<File> fileList = new TreeSet<File>(new Comparator() { - - /** {inheritDoc} */ - public int compare(Object o1, Object o2) { - // Need to make sure we read the child namespace before the base - // namespace. To do that, we sort the list in reverse order based - // on the length of the file name. If two filenames have the same - // length, then we need to do a lexigraphical comparison to make - // sure that the filenames themselves are different. - - File f1 = (File) o1; - File f2 = (File) o2; - int lenDif = f2.getAbsolutePath().length() - f1.getAbsolutePath().length(); - - if (lenDif != 0) { - return lenDif; - } - return f2.compareTo(f1); - } - }); - - /** - * public constructor. Rereads config entries every time it is called. - * - * @throws ConfigException error from the Configuration layers. the jakarta - * commons conf system just throws Exception, which makes it hard to react. - * sometimes it is an IOExceptions, sometimes a SAXParserException, - * sometimes a VindictiveException. so we just turn them into our own - * exception type and toss them up. as we discover ones we might - * meaningfully want to react to, we can specialize ConfigException and catch - * those - */ - public Configuration() throws ConfigException { - translators = TranslatorRegistry.getTranslators(); - addPath(getSearchConfigDir()); - addPath(getDefaultConfigFilePath()); - parseFiles(); - } - - /** - * Construct from a Reader. - * @param rdr Configuration data in Reader format. - * @throws ConfigException thrown if a problem reading the configuration - * occurs. - */ - public Configuration(Reader rdr) throws ConfigException { - translators = TranslatorRegistry.getTranslators(); - ReaderWrapper rdrwrap = new ReaderWrapper(rdr); - Properties props = new Properties(); - try { - parseStream(props, "search", rdrwrap); - } - catch (IOException ioe) { - throw new ConfigException(ioe.getMessage(), ioe); - } - } - - /** - * Add a path to the config object for parsing - * @param path The path to add - */ - public void addPath(String path) { - addFiles(path); - } - - /** - * static method to get the singleton Config option - * - * @return the config option - */ -// public static synchronized Config get() { -// if (singletonConfig == null) { -// singletonConfig = new Config(); -// } -// return singletonConfig; -// } - - private static String getRHNConfigDir() { - String confDir = System.getProperty(RHN_CONF_DIR_PROPERTY); - - if (StringUtils.isBlank(confDir)) { - confDir = DEFAULT_CONF_DIR; - } - return confDir; - } - - private static String getSearchConfigDir() { - String confDir = System.getProperty(SEARCH_CONF_DIR_PROPERTY); - - if (StringUtils.isBlank(confDir)) { - confDir = getRHNConfigDir() + "/search"; - } - return confDir; - } - - /** - * Get the path to the rhn.conf file we use. - * - * @return String path. - */ - public static String getDefaultConfigFilePath() { - return getRHNConfigDir() + "/rhn.conf"; - } - - /** - * Get the configuration entry for the given string name. If the value - * is null, then return the given defValue. defValue can be null as well. - * @param name name of property - * @param defValue default value for property if it is null. - * @return the value of the property with the given name, or defValue. - */ - public String getString(String name, String defValue) { - String ret = getString(name); - if (ret == null) { - if (logger.isDebugEnabled()) { - logger.debug("getString() - returning default value"); - } - ret = defValue; - } - return ret; - } - - /** - * Returns a map of the values with the given name. The value must be - * in the format of key:value otherwise, an empty map is returned. - * @param name name of property - * @return map of the values of the property matching k:v. - */ - public Map<String, String> getMap(String name) { - String value = getString(name); - Map retval = null; - if (value != null) { - retval = new HashMap<String, String>(); - String[] pairs = value.split(","); - for (int x = 0; x < pairs.length; x++) { - String[] nv = pairs[x].split(":"); - if (nv.length != 2) { - continue; - } - retval.put(nv[0].trim(), nv[1].trim()); - } - } - return retval; - } - - /** - * get the config entry for string s - * - * @param value string to get the value of - * @return the value - */ - public String getString(String value) { - if (logger.isDebugEnabled()) { - logger.debug("getString() - getString() called with: " + value); - } - if (value == null) { - return null; - } - - int lastDot = value.lastIndexOf("."); - String ns = ""; - String property = value; - if (lastDot > 0) { - property = value.substring(lastDot + 1); - ns = value.substring(0, lastDot); - } - if (logger.isDebugEnabled()) { - logger.debug("getString() - getString() -> Getting property: " + - property); - } - String result = configValues.getProperty(property); - if (logger.isDebugEnabled()) { - logger.debug("getString() - getString() -> result: " + result); - } - if (result == null) { - if (!"".equals(ns)) { - result = configValues.getProperty(ns + "." + property); - } - else { - for (int i = 0; i < prefixOrder.length; i++) { - result = configValues.getProperty(prefixOrder[i] + "." + property); - if (result != null) { - break; - } - } - } - } - if (logger.isDebugEnabled()) { - logger.debug("getString() - getString() -> returning: " + result); - } - - if (result == null || result.equals("")) { - return null; - } - - return stripComments(result); - } - - private String stripComments(String string) { - // check for null - if (string == null) { - return null; - } - - int breakpos = string.indexOf('#'); - - // check for comment in value - if (breakpos < 0) { - return string.trim(); - } - - while (breakpos > -1) { - if (breakpos == 0) { - string = ""; - break; - } - else if (string.charAt(breakpos - 1) == '\') { - breakpos = string.indexOf('#', breakpos + 1); - } - else { - string = string.substring(0, breakpos); - break; - } - } - - if (string.length() == 0) { - return string; - } - - return string.replaceAll("\\#", "#").trim(); - } - - /** - * get the config entry for string s - * - * @param s string to get the value of - * @return the value - */ - public int getInt(String s) { - return getInt(s, 0); - } - - /** - * get the config entry for string s, if no value is found - * return the defaultValue specified. - * - * @param s string to get the value of - * @param defaultValue Default value if entry is not found. - * @return the value - */ - public int getInt(String s, int defaultValue) { - Integer val = getInteger(s); - if (val == null) { - return defaultValue; - } - return val.intValue(); - } - - /** - * get the config entry for string s - * - * @param s string to get the value of - * @return the value - */ - public double getDouble(String s) { - return getDouble(s, 0.0); - } - - /** - * get the config entry for string s, if no value is found - * return the defaultValue specified. - * - * @param s string to get the value of - * @param defaultValue Default value if entry is not found. - * @return the value - */ - public double getDouble(String s, double defaultValue) { - String val = getString(s); - if (val == null) { - return defaultValue; - } - return new Double(val).doubleValue(); - } - - /** - * get the config entry for string s - * - * @param s string to get the value of - * @return the value - */ - public Integer getInteger(String s) { - String val = getString(s); - if (val == null) { - return null; - } - return new Integer(val); - } - - /** - * Parses a comma-delimited list of values as a java.util.List - * @param name config entry name - * @return instance of java.util.List populated with config values - */ - public List getList(String name) { - List retval = new LinkedList(); - String[] vals = getStringArray(name); - if (vals != null) { - retval.addAll(Arrays.asList(vals)); - } - return retval; - } - - /** - * get the config entry for string s - * - * @param s string to get the value of - * @return the value - */ - public String[] getStringArray(String s) { - if (s == null) { - return null; - } - String value = getString(s); - - if (value == null) { - return null; - } - - return value.split(","); - } - - /** - * get the config entry for string name - * - * @param name string to set the value of - * @param value new value - * @return the previous value of the property - */ - public String setString(String name, String value) { - return (String) configValues.setProperty(name, value); - } - - /** - * get the config entry for string s - * - * @param s string to get the value of - * @return the value - */ - public boolean getBoolean(String s) { - String value = getString(s); - if (logger.isDebugEnabled()) { - logger.debug("getBoolean() - " + s + " is : " + value); - } - if (value == null) { - return false; - } - - //need to check the possible true values - // tried to use BooleanUtils, but that didn't - // get the job done for an integer as a String. - - - for (int i = 0; i < TRUE_VALUES.length; i++) { - if (TRUE_VALUES[i].equalsIgnoreCase(value)) { - if (logger.isDebugEnabled()) { - logger.debug("getBoolean() - Returning true: " + value); - } - return true; - } - } - - return false; - } - - /** - * set the config entry for string name - * @param s string to set the value of - * @param b new value - */ - public void setBoolean(String s, String b) { - // need to check the possible true values - // tried to use BooleanUtils, but that didn't - // get the job done for an integer as a String. - for (int i = 0; i < TRUE_VALUES.length; i++) { - if (TRUE_VALUES[i].equalsIgnoreCase(b)) { - configValues.setProperty(s, "1"); - - // get out we're done here - return; - } - } - configValues.setProperty(s, "0"); - } - - private void addFiles(String path) { - File f = new File(path); - - if (f.isDirectory()) { - // bugzilla: 154517; only add items that end in .conf - File[] files = f.listFiles(); - for (int i = 0; i < files.length; i++) { - if (files[i].getName().endsWith((".conf"))) { - fileList.add(files[i]); - } - } - } - else { - fileList.add(f); - } - } - - private String makeNamespace(File f) { - String ns = f.getName(); - - // This is really hokey, but it works. Basically, rhn.conf doesn't - // match the standard rhn_foo.conf convention. So, to create the - // namespace, we first special case rhn.* - if (ns.startsWith("rhn.")) { - return ""; - } - - ns = ns.replaceFirst("rhn_", ""); - ns = ns.substring(0, ns.lastIndexOf('.')); - ns = ns.replaceAll("_", "."); - return ns; - } - - private void parseStream(Properties props, String namespace, InputStream is) - throws IOException { - - props.load(is); - - // loop through all of the config values in the properties file - // making sure the prefix is there. - Properties newProps = new Properties(); - for (Iterator j = props.keySet().iterator(); j.hasNext();) { - String key = (String) j.next(); - String newKey = key; - if (!key.startsWith(namespace)) { - newKey = namespace + "." + key; - } - logger.debug("Adding: " + newKey + ": " + props.getProperty(key)); - // translate the original key - newKey = translateKey(key); - newProps.put(newKey, props.getProperty(key)); - } - configValues.putAll(newProps); - } - - /** - * Parse all of the added files. - */ - public void parseFiles() { - for (Iterator i = fileList.iterator(); i.hasNext();) { - File curr = (File) i.next(); - parseFile(curr); - } - } - - private void parseFile(File file) { - Properties props = new Properties(); - String ns = makeNamespace(file); - logger.debug("Adding namespace: " + ns + " for file: " + - file.getAbsolutePath()); - - try { - parseStream(props, ns, new FileInputStream(file)); - } - catch (IOException e) { - logger.error("Could not parse file" + file, e); - } - } - - /** - * Returns a subset of the properties for the given namespace. This is - * not a particularly fast method and should be used only at startup or - * some other discreet time. Repeated calls to this method are guaranteed - * to be slow. - * @param namespace Namespace of properties to be returned. - * @return subset of the properties that begin with the given namespace. - */ - public Properties getNamespaceProperties(String namespace) { - Properties prop = new Properties(); - for (Iterator i = configValues.keySet().iterator(); i.hasNext();) { - String key = (String) i.next(); - if (key.startsWith(namespace)) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for key: [" + key + "]"); - } - prop.put(key, configValues.getProperty(key)); - } - } - return prop; - } - - private String translateKey(String key) { - for (Iterator itr = translators.iterator(); itr.hasNext();) { - KeyTranslator trans = (KeyTranslator) itr.next(); - if (trans.shouldTranslate(key)) { - key = trans.translateKey(key); - } - } - - return key; - } - - private static class ReaderWrapper extends InputStream { - private Reader rdr; - - public ReaderWrapper(Reader reader) { - rdr = reader; - } - - /** - * {@inheritDoc} - */ - @Override - public int read() throws IOException { - return rdr.read(); - } - - @Override - public void close() throws IOException { - super.close(); - rdr.close(); - } - } -} - diff --git a/search-server/src/java/com/redhat/satellite/search/config/KeyTranslator.java b/search-server/src/java/com/redhat/satellite/search/config/KeyTranslator.java deleted file mode 100644 index 26f442a..0000000 --- a/search-server/src/java/com/redhat/satellite/search/config/KeyTranslator.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.config; - - -/** - * Translator - * @version $Rev$ - */ -public interface KeyTranslator { - - /** - * Translates the given key into something else. That something else is - * determined by the specific implementation. The value for the given key - * should not be affected. - * @param key key to be translated. - * @return The translated key. - */ - String translateKey(String key); - - /** - * Returns true if the key should be translated. - * @param key to be translated - * @return true if the key should be translated. - */ - boolean shouldTranslate(String key); -} diff --git a/search-server/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java b/search-server/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java deleted file mode 100644 index 9ac2283..0000000 --- a/search-server/src/java/com/redhat/satellite/search/config/tests/ConfigurationTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.config.tests; - -import com.redhat.satellite.search.config.Configuration; - -import java.io.BufferedReader; -import java.io.StringReader; -import java.util.List; -import java.util.Map; - -import junit.framework.TestCase; - - -/** - * ConfigurationTest - * @version $Rev$ - */ -public class ConfigurationTest extends TestCase { - - private Configuration config; - - @Override - protected void setUp() throws Exception { - StringBuilder builder = new StringBuilder(); - builder.append("search.boolean=true\n"); - builder.append("search.index_work_dir=/tmp/search-server-test\n"); - builder.append("search.rpc_port=2828\n"); - builder.append("search.list=item1,item2, item3\n"); - builder.append("hibernate.connection.username=rhnsat\n"); - builder.append("search.rpc_handlers="); - builder.append( - "index:com.redhat.satellite.search.rpc.handlers.IndexHandler\n\n"); - StringReader sr = new StringReader(builder.toString()); - config = new Configuration(new BufferedReader(sr)); - } - - public void testGetValue() { - assertEquals("bar", config.getString("foo", "bar")); - assertEquals( - "index:com.redhat.satellite.search.rpc.handlers.IndexHandler", - config.getString("search.rpc_handlers", "")); - assertNull(config.getString(null, null)); - assertEquals("", config.getString(null, "")); - } - - public void testGetValueAsInt() { - assertEquals(2828, config.getInt("search.rpc_port")); - assertEquals(2828, config.getInt("search.rpc_port", 0)); - assertEquals(10, config.getInt("search.nothere", 10)); - assertEquals(10, config.getInt(null, 10)); - try { - config.getInt("search.rpc_handlers", 0); - fail("getValueAsInt didn't throw exception"); - } - catch (Exception e) { - // expected - } - } - - public void testGetValuesAsMap() { - assertNull(config.getMap("search.notthere")); - assertNull(config.getMap(null)); - - Map<String, String> map = config.getMap("search.rpc_handlers"); - assertNotNull(map); - assertEquals("com.redhat.satellite.search.rpc.handlers.IndexHandler", - map.get("index")); - - map = config.getMap("search.rpc_port"); - assertNotNull(map); - assertTrue(map.keySet().isEmpty()); - } - - public void testGetValues() { - assertNotNull(config.getList("search.notthere")); - assertTrue(config.getList("search.notthere").isEmpty()); - assertNotNull(config.getList(null)); - assertTrue(config.getList(null).isEmpty()); - - List<String> list = config.getList("search.list"); - assertNotNull(list); - assertEquals(3, list.size()); - assertEquals("item1", list.get(0)); - assertEquals("item2", list.get(1)); - assertEquals(" item3", list.get(2)); - } - - public void testTranslation() { - assertNull(config.getString("hibernate.connection.username")); - assertEquals("rhnsat", config.getString("search.connection.username")); - } - - public void testGetBoolean() { - assertTrue(config.getBoolean("search.boolean")); - config.setBoolean("search.boolean", "false"); - assertFalse(config.getBoolean("search.boolean")); - } - - public void testGetDefaultConfig() { - String confDir = System.getProperty("rhn.config.dir"); - if (confDir == null || "".equals(confDir)) { - confDir = "/etc/rhn"; - } - assertEquals(confDir + "/rhn.conf", - Configuration.getDefaultConfigFilePath()); - } - - public void testSetString() { - config.setString("search.stringvalue", "foobar"); - assertEquals("foobar", config.getString("search.stringvalue")); - } - - public void testGetDouble() { - config.setString("search.double", ".30"); - assertEquals(0.0, config.getDouble("better.not.be.a.double")); - assertEquals(.30, config.getDouble("search.double")); - - config.setString("search.double", "30"); - assertEquals(30.0, config.getDouble("search.double")); - - config.setString("search.double", ""); - assertEquals(0.0, config.getDouble("search.double")); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java b/search-server/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java deleted file mode 100644 index b3eb094..0000000 --- a/search-server/src/java/com/redhat/satellite/search/config/translator/HibernateToSearchTranslator.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.config.translator; - -import com.redhat.satellite.search.config.KeyTranslator; - - -/** - * HibernateToSearchTranslator - * @version $Rev$ - */ -public class HibernateToSearchTranslator implements KeyTranslator { - - /** - * {@inheritDoc} - */ - public String translateKey(String key) { - if (key.equals("hibernate.connection.username") || - key.equals("hibernate.connection.password") || - key.equals("hibernate.connection.driver_class") || - key.equals("hibernate.connection.url")) { - return key.replaceAll("hibernate", "search"); - } - - return key; - } - - /** - * {@inheritDoc} - */ - public boolean shouldTranslate(String key) { - return (key.equals("hibernate.connection.username") || - key.equals("hibernate.connection.password") || - key.equals("hibernate.connection.driver_class") || - key.equals("hibernate.connection.url")); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java b/search-server/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java deleted file mode 100644 index 4564812..0000000 --- a/search-server/src/java/com/redhat/satellite/search/config/translator/TranslatorRegistry.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.config.translator; - -import com.redhat.satellite.search.config.KeyTranslator; - -import java.util.LinkedList; -import java.util.List; - - -/** - * TranslatorRegistry - * @version $Rev$ - */ -public class TranslatorRegistry { - - private TranslatorRegistry() { - // hide the default constructor - } - - private static final List<Class> TRANSLATOR_CLASSES; - static { - TRANSLATOR_CLASSES = new LinkedList<Class>(); - TRANSLATOR_CLASSES.add(HibernateToSearchTranslator.class); - } - - private static List<KeyTranslator> translators = new LinkedList<KeyTranslator>(); - - /** - * Returns the list of all available translators. - * @return the list of all available translators. - */ - public static List<KeyTranslator> getTranslators() { - if (translators.isEmpty() && !TRANSLATOR_CLASSES.isEmpty()) { - for (Class clazz : TRANSLATOR_CLASSES) { - try { - Object s = clazz.newInstance(); - translators.add((KeyTranslator)s); - } - catch (Exception e) { - e.printStackTrace(System.out); - } - } - } - - return translators; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/Connection.java b/search-server/src/java/com/redhat/satellite/search/db/Connection.java deleted file mode 100644 index 5325a56..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/Connection.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.db; - -import com.ibatis.sqlmap.client.SqlMapSession; - -import org.apache.log4j.Logger; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -/** - * Simplified connection class for executing arbitrary SQL - * - * @version $Rev$ - */ -public class Connection { - private static Logger log = Logger.getLogger(Connection.class); - - private SqlMapSession session; - - /** - * Constructor - * @param sess iBatis session to use - */ - public Connection(SqlMapSession sess) { - session = sess; - } - - /** - * Close down all resources - * @throws SQLException something bad happened - */ - public void close() throws SQLException { - session.close(); - } - - /** - * Executes a query - * @param statement string of valid SQL - * @param handler called on each row - * @throws SQLException something bad happened - */ - public void executeQuery(String statement, ResultHandler handler) throws SQLException { - java.sql.Connection cn = null; - Statement stmt = null; - ResultSet rs = null; - try { - log.warn("start transaction"); - session.startTransaction(); - log.warn("get current connection"); - cn = session.getCurrentConnection(); - log.warn("creating statement"); - stmt = cn.createStatement(); - log.warn("execute query"); - rs = stmt.executeQuery(statement); - log.warn("process resultset"); - while (rs.next()) { - handler.handleRow(rs); - } - } - finally { - try { - log.warn("commit transaction"); - session.commitTransaction(); - } - finally { - try { - if (rs != null) { - log.warn("close RS"); - rs.close(); - } - } - finally { - if (stmt != null) { - log.warn("close STMT"); - stmt.close(); - } - } - } - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/DatabaseManager.java b/search-server/src/java/com/redhat/satellite/search/db/DatabaseManager.java deleted file mode 100644 index 93ad0a9..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/DatabaseManager.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.db; - -import com.redhat.satellite.search.config.Configuration; - -import com.ibatis.sqlmap.client.SqlMapClient; -import com.ibatis.sqlmap.client.SqlMapClientBuilder; -import com.ibatis.sqlmap.client.SqlMapSession; - -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Properties; - -/** - * Manages DB activity - connections, running queries, etc - * @version $Rev$ - */ -public class DatabaseManager { - - private SqlMapClient client = null; - - /** - * Constructor - * @param config - * @throws IOException - */ - public DatabaseManager(Configuration config) throws IOException { - String configPath = config.getString("search.db_config_path", - "classpath"); - Reader reader = null; - if (configPath.equals("classpath")) { - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - InputStream stream = cl - .getResourceAsStream("com/redhat/satellite/search/db/config.xml"); - if (stream == null) { - throw new IllegalArgumentException( - "com/redhat/satellite/search/db/" + - "config.xml resource missing"); - } - reader = new InputStreamReader(stream); - } - else { - reader = new FileReader(configPath); - } - Properties overrides = config.getNamespaceProperties("search"); - client = SqlMapClientBuilder.buildSqlMapClient(reader, overrides); - } - - /** - * Open a named select query - * @param <T> type returned by query - * @param name name of query - * @return query object - */ - public <T> Query<T> getQuery(String name) { - SqlMapSession session = client.openSession(); - return new Query<T>(session, name); - } - - /** - * Open a named write (insert, update, delete) query - * @param name of query - * @return query object - */ - public WriteQuery getWriterQuery(String name) { - SqlMapSession session = client.openSession(); - return new WriteQuery(session, name); - } - - /** - * Opens a direct DB connection - * @return connection object - */ - public Connection getConnection() { - return new Connection(client.openSession()); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/Query.java b/search-server/src/java/com/redhat/satellite/search/db/Query.java deleted file mode 100644 index 57c209e..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/Query.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.db; - -import com.ibatis.sqlmap.client.SqlMapSession; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -/** - * Named Select query - * @param <T> type returned by query - * - * @version $Rev$ - */ -public class Query<T> { - - private SqlMapSession session; - private String queryName; - - Query(SqlMapSession sessionIn, String queryNameIn) { - session = sessionIn; - queryName = queryNameIn; - } - - /** - * Close query and associated resources - * @throws SQLException something bad happened - */ - public void close() throws SQLException { - try { - try { - session.commitTransaction(); - } - catch (SQLException e) { - session.endTransaction(); - } - } - finally { - session.close(); - } - } - - /** - * Load list of objects from query - * @param param query param - * @return list of T - * @throws SQLException something bad happened - */ - @SuppressWarnings("unchecked") - public List<T> loadList(Object param) throws SQLException { - List r = session.queryForList(queryName, param); - List<Object> results = (List<Object>)r; - List<T> retval = new ArrayList<T>(results.size()); - for (Object item : results) { - retval.add((T)item); - } - return retval; - } - - /** - * Load single object from query - * @param param query apram - * @return T - * @throws SQLException something bad happened - */ - @SuppressWarnings("unchecked") - public T load(Object param) throws SQLException { - return (T)session.queryForObject(queryName, param); - } - - /** - * Load single object from query - * @return T - * @throws SQLException something bad happened - */ - @SuppressWarnings("unchecked") - public T load() throws SQLException { - return (T)session.queryForObject(queryName); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/ResultHandler.java b/search-server/src/java/com/redhat/satellite/search/db/ResultHandler.java deleted file mode 100644 index caf86cb..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/ResultHandler.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - - -package com.redhat.satellite.search.db; - -import java.sql.ResultSet; -import java.sql.SQLException; - -/** - * Callback interface for Connection-based queries - * - * @version $Rev$ - */ -public interface ResultHandler { - - /** - * Called for each row in a resultset - * @param rs resultset - * @throws SQLException something bad happened - */ - void handleRow(ResultSet rs) throws SQLException; -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/WriteQuery.java b/search-server/src/java/com/redhat/satellite/search/db/WriteQuery.java deleted file mode 100644 index c394316..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/WriteQuery.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.db; - -import com.ibatis.sqlmap.client.SqlMapSession; - -import java.sql.SQLException; - -/** - * Query for writing (insert, update, delete) to a database - * - * @version $Rev$ - */ -public class WriteQuery { - - private SqlMapSession session; - private String queryName; - - WriteQuery(SqlMapSession sessionIn, String queryNameIn) { - session = sessionIn; - queryName = queryNameIn; - } - - /** - * Close query and all associated resources - * @throws SQLException something bad happened - */ - public void close() throws SQLException { - try { - try { - session.commitTransaction(); - } - catch (SQLException e) { - session.endTransaction(); - } - } - finally { - session.close(); - } - } - - /** - * Execute update query - * @param param query param - * @return number of rows updated - * @throws SQLException something bad happened - */ - public int update(Object param) throws SQLException { - return session.update(queryName, param); - } - - /** - * Execute delete query - * @param param query param - * @return number of rows updated - * @throws SQLException something bad happened - */ - public int delete(Object param) throws SQLException { - return session.delete(queryName, param); - } - - /** - * Execute insert query - * @param param query param - * @throws SQLException something bad happened - */ - public void insert(Object param) throws SQLException { - session.insert(queryName, param); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/models/Errata.java b/search-server/src/java/com/redhat/satellite/search/db/models/Errata.java deleted file mode 100644 index 2a6d170..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/models/Errata.java +++ /dev/null @@ -1,323 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.db.models; - - -/** - * Errata - * @version $Rev$ - */ -public class Errata { - private long id; - private String advisory; - private String advisoryType; - private String advisoryName; - private long advisoryRel; - private String product; - private String description; - private String synopsis; - private String topic; - private String solution; - private String issueDate; - private String updateDate; - private String notes; - private String orgId; - private String created; - private String modified; - private String lastModified; - - /** - * Returns a string representation of the errata - * @return string representation of the errata - */ - public String toString() { - StringBuffer text = new StringBuffer("Errata<" + id + ", " + product + ">: "); - text.append(advisoryType + ", " + advisoryName + ", " + advisory); - text.append(", " + topic + ", " + synopsis); - return text.toString(); - } - - /** - * Returns the erratum's id. - * @return the erratum's id. - */ - public long getId() { - return id; - } - - /** - * The erratum's id. - * @param idIn erratum id. - */ - public void setId(long idIn) { - id = idIn; - } - - /** - * Returns the advisory. - * @return the advisory. - */ - public String getAdvisory() { - return advisory; - } - - /** - * Sets the advisory. - * @param advisoryIn - */ - public void setAdvisory(String advisoryIn) { - advisory = advisoryIn; - } - - /** - * Returns advisory name - * @return advisory name - */ - public String getAdvisoryName() { - return advisoryName; - } - - /** - * Sets advisory name - * @param advisoryNameIn new advisory name - */ - public void setAdvisoryName(String advisoryNameIn) { - advisoryName = advisoryNameIn; - } - - /** - * Returns advisory release - * @return advisory release - */ - public long getAdvisoryRel() { - return advisoryRel; - } - - /** - * Sets advisory release - * @param advisoryRelIn advisory release - */ - public void setAdvisoryRel(long advisoryRelIn) { - advisoryRel = advisoryRelIn; - } - - /** - * Returns advisory type - * @return advisory type - */ - public String getAdvisoryType() { - return advisoryType; - } - - /** - * Set advisory type - * @param advisoryTypeIn new advisory type - */ - public void setAdvisoryType(String advisoryTypeIn) { - advisoryType = advisoryTypeIn; - } - - /** - * Returns created date as a string - * @return created date as a string - */ - public String getCreated() { - return created; - } - - /** - * Sets the created date (expecting string format) - * @param createdIn string formatted date. - */ - public void setCreated(String createdIn) { - created = createdIn; - } - - /** - * Returns the errata description - * @return the errata description - */ - public String getDescription() { - return description; - } - - /** - * Sets the description - * @param descriptionIn description of errata - */ - public void setDescription(String descriptionIn) { - description = descriptionIn; - } - - /** - * Returns the erratum's issue date as a string. - * @return the erratum's issue date as a string. - */ - public String getIssueDate() { - return issueDate; - } - - /** - * Sets the issue date (expecting string format). - * @param issueDateIn string formatted date. - */ - public void setIssueDate(String issueDateIn) { - issueDate = issueDateIn; - } - - /** - * Returns the last time the erratum was modified as a string. - * @return the last time the erratum was modified as a string. - */ - public String getLastModified() { - return lastModified; - } - - /** - * sets the last modified date as a string. - * @param lastModifiedIn string formatted last modified date. - */ - public void setLastModified(String lastModifiedIn) { - lastModified = lastModifiedIn; - } - - /** - * Returns the modified date as a string. - * @return the modified date as a string. - */ - public String getModified() { - return modified; - } - - /** - * Sets the modified date as a string. - * @param modifiedIn string formatted date. - */ - public void setModified(String modifiedIn) { - modified = modifiedIn; - } - - /** - * Returns notes associated with the errataum. - * @return notes associated with the errataum. - */ - public String getNotes() { - return notes; - } - - /** - * Sets the notes for the erratum. - * @param notesIn notes. - */ - public void setNotes(String notesIn) { - notes = notesIn; - } - - /** - * The org id owning the errata. - * @return org id owning the errata. - */ - public String getOrgId() { - return orgId; - } - - /** - * Sets the orgid owning the errata. - * @param orgIdIn owning orgid. - */ - public void setOrgId(String orgIdIn) { - orgId = orgIdIn; - } - - /** - * Products affected. - * @return affected products. - */ - public String getProduct() { - return product; - } - - /** - * sets the product. - * @param productIn product - */ - public void setProduct(String productIn) { - product = productIn; - } - - /** - * Returns what the errata proposed solution. - * @return what the errata proposed solution. - */ - public String getSolution() { - return solution; - } - - /** - * Proposed solution - * @param solutionIn proposed solution. - */ - public void setSolution(String solutionIn) { - solution = solutionIn; - } - - /** - * Returns erratum's synopsis. - * @return erratum's synopsis. - */ - public String getSynopsis() { - return synopsis; - } - - /** - * Sets the erratum's synopsis. - * @param synopsisIn synopsis. - */ - public void setSynopsis(String synopsisIn) { - synopsis = synopsisIn; - } - - /** - * returns erratum topic. - * @return erratum topic. - */ - public String getTopic() { - return topic; - } - - /** - * Sets the erratum's topic. - * @param topicIn topic - */ - public void setTopic(String topicIn) { - topic = topicIn; - } - - /** - * Returns the update date as a string. - * @return the update date as a string. - */ - public String getUpdateDate() { - return updateDate; - } - - /** - * Sets the update date as a string. - * @param updateDateIn string formatted date. - */ - public void setUpdateDate(String updateDateIn) { - updateDate = updateDateIn; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/models/GenericRecord.java b/search-server/src/java/com/redhat/satellite/search/db/models/GenericRecord.java deleted file mode 100644 index fc51f60..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/models/GenericRecord.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.db.models; - -/** - * Server - * @version $Rev$ - */ -public abstract class GenericRecord { - protected long id = 0; - /** - * @return the id - */ - public long getId() { - return id; - } - /** - * @param idIn the id to set - */ - public void setId(long idIn) { - this.id = idIn; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java b/search-server/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java deleted file mode 100644 index e4e82b7..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/models/HardwareDevice.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.db.models; - -/** - * Server - * @version $Rev$ - */ -public class HardwareDevice extends GenericRecord { - - private long serverId; - private String classInfo; - private String bus; - private long detached; - private String device; - private String driver; - private String description; - private String vendorId; - private String deviceId; - private String subVendorId; - private String subDeviceId; - private long pciType; - - /** - * @return the serverId - */ - public long getServerId() { - return serverId; - } - /** - * @param serverIdIn the serverId to set - */ - public void setServerId(long serverIdIn) { - this.serverId = serverIdIn; - } - /** - * @return the classInfo - */ - public String getClassInfo() { - return classInfo; - } - /** - * @param classInfoIn the classInfo to set - */ - public void setClassInfo(String classInfoIn) { - this.classInfo = classInfoIn; - } - /** - * @return the bus - */ - public String getBus() { - return bus; - } - /** - * @param busIn the bus to set - */ - public void setBus(String busIn) { - this.bus = busIn; - } - /** - * @return the detached - */ - public long getDetached() { - return detached; - } - /** - * @param detachedIn the detached to set - */ - public void setDetached(long detachedIn) { - this.detached = detachedIn; - } - /** - * @return the device - */ - public String getDevice() { - return device; - } - /** - * @param deviceIn the device to set - */ - public void setDevice(String deviceIn) { - this.device = deviceIn; - } - /** - * @return the driver - */ - public String getDriver() { - return driver; - } - /** - * @param driverIn the driver to set - */ - public void setDriver(String driverIn) { - this.driver = driverIn; - } - /** - * @return the description - */ - public String getDescription() { - return description; - } - /** - * @param descriptionIn the description to set - */ - public void setDescription(String descriptionIn) { - this.description = descriptionIn; - } - /** - * @return the vendorId - */ - public String getVendorId() { - return vendorId; - } - /** - * @param vendorIdIn the vendorId to set - */ - public void setVendorId(String vendorIdIn) { - this.vendorId = vendorIdIn; - } - /** - * @return the deviceId - */ - public String getDeviceId() { - return deviceId; - } - /** - * @param deviceIdIn the deviceId to set - */ - public void setDeviceId(String deviceIdIn) { - this.deviceId = deviceIdIn; - } - /** - * @return the subVendorId - */ - public String getSubVendorId() { - return subVendorId; - } - /** - * @param subVendorIdIn the subVendorId to set - */ - public void setSubVendorId(String subVendorIdIn) { - this.subVendorId = subVendorIdIn; - } - /** - * @return the subDeviceId - */ - public String getSubDeviceId() { - return subDeviceId; - } - /** - * @param subDeviceIdIn the subDeviceId to set - */ - public void setSubDeviceId(String subDeviceIdIn) { - this.subDeviceId = subDeviceIdIn; - } - /** - * @return the pciType - */ - public long getPciType() { - return pciType; - } - /** - * @param pciTypeIn the pciType to set - */ - public void setPciType(long pciTypeIn) { - this.pciType = pciTypeIn; - } - - - -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/models/RhnPackage.java b/search-server/src/java/com/redhat/satellite/search/db/models/RhnPackage.java deleted file mode 100644 index 929b381..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/models/RhnPackage.java +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.db.models; - -/** - * Simple Package DTO - * @version $Rev$ - */ -public class RhnPackage { - - private long id; - private String name; - private String epoch; - private String version; - private String release; - private String arch; - private String description; - private String summary; - - /** - * Getter for pkg id - * @return pkg id - */ - public long getId() { - return id; - } - - /** - * Setter for pkg id - * @param idIn pkg id - */ - public void setId(long idIn) { - this.id = idIn; - } - - /** - * Getter for pkg name - * @return pkg name - */ - public String getName() { - return name; - } - - /** - * Setter for pkg name - * @param nameIn pkg name - */ - public void setName(String nameIn) { - this.name = nameIn; - } - - /** - * Getter for pkg epoch - * @return pkg epoch - */ - public String getEpoch() { - return epoch; - } - - /** - * Setter for pkg epoch - * @param epochIn pkg epoch - */ - public void setEpoch(String epochIn) { - this.epoch = epochIn; - } - - /** - * Getter for pkg version - * @return pkg version - */ - public String getVersion() { - return version; - } - - /** - * Setter for pkg version - * @param versionIn - */ - public void setVersion(String versionIn) { - this.version = versionIn; - } - - /** - * Getter for pkg release - * @return pkg release - */ - public String getRelease() { - return release; - } - - /** - * Setter for pkg release - * @param releaseIn pkg release - */ - public void setRelease(String releaseIn) { - this.release = releaseIn; - } - - /** - * Getter for pkg arch - * @return pkg arch - */ - public String getArch() { - return arch; - } - - /** - * Setter for pkg arch - * @param archIn pkg arch - */ - public void setArch(String archIn) { - this.arch = archIn; - } - - /** - * Setter for pkg description - * @param desc pkg description - */ - public void setDescription(String desc) { - description = desc; - } - - /** - * Getter for pkg description - * @return pkg description - */ - public String getDescription() { - return description; - } - - /** - * Setter for pkg summary - * @param summaryIn pkg summary - */ - public void setSummary(String summaryIn) { - summary = summaryIn; - } - - /** - * Getter for pkg summary - * @return pkg summary - */ - public String getSummary() { - return summary; - } - - /** - * Getter for "pretty" versino - * @return <version>-<release> - */ - public String getPrettyVersion() { - return version + "-" + release; - } - - /** - * Reconstructs filename from various parts of pkg metadata - * @return <name>-<version>-<release>.<arch> - */ - public String getFileName() { - return name + "-" + version + "-" + release + "." + arch; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/models/Server.java b/search-server/src/java/com/redhat/satellite/search/db/models/Server.java deleted file mode 100644 index 4775655..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/models/Server.java +++ /dev/null @@ -1,698 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.db.models; - -import java.util.Date; - -import org.apache.lucene.document.DateTools; -import org.apache.lucene.document.NumberTools; - - -/** - * Server - * @version $Rev$ - */ -public class Server extends GenericRecord { - - private String name; - private String info; - private String description; - - /* Location */ - private String machine; - private String rack; - private String room; - private String building; - private String address1; - private String address2; - private String city; - private String state; - private String country; - - /* Network Info */ - private String hostname; - private String ipaddr; - - /* DMI Info */ - private String dmiVendor; - private String dmiSystem; - private String dmiProduct; - private String dmiBiosVendor; - private String dmiBiosVersion; - private String dmiBiosRelease; - private String dmiAsset; - private String dmiBoard; - - /* CPU Info */ - private String cpuBogoMIPS; - private String cpuCache; - private String cpuFamily; - private String cpuMhz; - private String cpuStepping; - private String cpuFlags; - private String cpuModel; - private String cpuVersion; - private String cpuVendor; - private String cpuNumberOfCpus; - private String cpuAcpiVersion; - private String cpuApic; - private String cpuApmVersion; - private String cpuChipset; - - private String checkin; - private String registered; - - private String ram; - private String swap; - - private String runningKernel; - - /** - * @return the name - */ - public String getName() { - return name; - } - /** - * @param nameIn the name to set - */ - public void setName(String nameIn) { - this.name = nameIn; - } - - /** - * @return the info - */ - public String getInfo() { - return info; - } - - /** - * @param infoIn the info to set - */ - public void setInfo(String infoIn) { - this.info = infoIn; - } - - /** - * @return the description - */ - public String getDescription() { - return description; - } - - /** - * @param descriptionIn the description to set - */ - public void setDescription(String descriptionIn) { - this.description = descriptionIn; - } - - /** - * @return the machine - */ - public String getMachine() { - return machine; - } - - /** - * @param machineIn the machine to set - */ - public void setMachine(String machineIn) { - this.machine = machineIn; - } - - /** - * @return the rack - */ - public String getRack() { - return rack; - } - - /** - * @param rackIn the rack to set - */ - public void setRack(String rackIn) { - this.rack = rackIn; - } - - /** - * @return the room - */ - public String getRoom() { - return room; - } - - /** - * @param roomIn the room to set - */ - public void setRoom(String roomIn) { - this.room = roomIn; - } - - /** - * @return the building - */ - public String getBuilding() { - return building; - } - - /** - * @param buildingIn the building to set - */ - public void setBuilding(String buildingIn) { - this.building = buildingIn; - } - - /** - * @return the address1 - */ - public String getAddress1() { - return address1; - } - - /** - * @param address1In the address1 to set - */ - public void setAddress1(String address1In) { - this.address1 = address1In; - } - - /** - * @return the address2 - */ - public String getAddress2() { - return address2; - } - - /** - * @param address2In the address2 to set - */ - public void setAddress2(String address2In) { - this.address2 = address2In; - } - - /** - * @return the city - */ - public String getCity() { - return city; - } - - /** - * @param cityIn the city to set - */ - public void setCity(String cityIn) { - this.city = cityIn; - } - - /** - * @return the state - */ - public String getState() { - return state; - } - - /** - * @param stateIn the state to set - */ - public void setState(String stateIn) { - this.state = stateIn; - } - - /** - * @return the country - */ - public String getCountry() { - return country; - } - - /** - * @param countryIn the country to set - */ - public void setCountry(String countryIn) { - this.country = countryIn; - } - - /** - * @return the hostname - */ - public String getHostname() { - return hostname; - } - - /** - * @param hostnameIn the hostname to set - */ - public void setHostname(String hostnameIn) { - this.hostname = hostnameIn; - } - - /** - * @return the ipaddr - */ - public String getIpaddr() { - return ipaddr; - } - - /** - * @param ipaddrIn the ipaddr to set - */ - public void setIpaddr(String ipaddrIn) { - this.ipaddr = ipaddrIn; - } - - /** - * @return the dmiVendor - */ - public String getDmiVendor() { - return dmiVendor; - } - - /** - * @param dmiVendorIn the dmiVendor to set - */ - public void setDmiVendor(String dmiVendorIn) { - this.dmiVendor = dmiVendorIn; - } - - /** - * @return the dmiSystem - */ - public String getDmiSystem() { - return dmiSystem; - } - - /** - * @param dmiSystemIn the dmiSystem to set - */ - public void setDmiSystem(String dmiSystemIn) { - this.dmiSystem = dmiSystemIn; - } - - /** - * @return the dmiProduct - */ - public String getDmiProduct() { - return dmiProduct; - } - - /** - * @param dmiProductIn the dmiProduct to set - */ - public void setDmiProduct(String dmiProductIn) { - this.dmiProduct = dmiProductIn; - } - - /** - * @return the dmiBiosVendor - */ - public String getDmiBiosVendor() { - return dmiBiosVendor; - } - - /** - * @param dmiBiosVendorIn the dmiBiosVendor to set - */ - public void setDmiBiosVendor(String dmiBiosVendorIn) { - this.dmiBiosVendor = dmiBiosVendorIn; - } - - /** - * @return the dmiBiosVersion - */ - public String getDmiBiosVersion() { - return dmiBiosVersion; - } - - /** - * @param dmiBiosVersionIn the dmiBiosVersion to set - */ - public void setDmiBiosVersion(String dmiBiosVersionIn) { - this.dmiBiosVersion = dmiBiosVersionIn; - } - - /** - * @return the dmiBiosRelease - */ - public String getDmiBiosRelease() { - return dmiBiosRelease; - } - - /** - * @param dmiBiosReleaseIn the dmiBiosRelease to set - */ - public void setDmiBiosRelease(String dmiBiosReleaseIn) { - this.dmiBiosRelease = dmiBiosReleaseIn; - } - - /** - * @return the dmiAsset - */ - public String getDmiAsset() { - return dmiAsset; - } - - /** - * @param dmiAssetIn the dmiAsset to set - */ - public void setDmiAsset(String dmiAssetIn) { - this.dmiAsset = dmiAssetIn; - } - - /** - * @return the dmiBoard - */ - public String getDmiBoard() { - return dmiBoard; - } - - /** - * @param dmiBoardIn the dmiBoard to set - */ - public void setDmiBoard(String dmiBoardIn) { - this.dmiBoard = dmiBoardIn; - } - - /** - * @return the cpuBogoMIPS - */ - public String getCpuBogoMIPS() { - return cpuBogoMIPS; - } - - /** - * @param cpuBogoMIPSIn the cpuBogoMIPS to set - */ - public void setCpuBogoMIPS(String cpuBogoMIPSIn) { - if (cpuBogoMIPSIn != null) { - Float f = Float.parseFloat(cpuBogoMIPSIn); - this.cpuBogoMIPS = NumberTools.longToString(f.longValue()); - } - else { - this.cpuBogoMIPS = null; - } - } - - /** - * @return the cpuCache - */ - public String getCpuCache() { - return cpuCache; - } - - /** - * @param cpuCacheIn the cpuCache to set - */ - public void setCpuCache(String cpuCacheIn) { - this.cpuCache = cpuCacheIn; - } - - /** - * @return the cpuFamily - */ - public String getCpuFamily() { - return cpuFamily; - } - - /** - * @param cpuFamilyIn the cpuFamily to set - */ - public void setCpuFamily(String cpuFamilyIn) { - this.cpuFamily = cpuFamilyIn; - } - - /** - * @return the cpuMhz - */ - public String getCpuMhz() { - return cpuMhz; - } - - /** - * @param cpuMhzIn the cpuMhz to set - */ - public void setCpuMhz(String cpuMhzIn) { - if (cpuMhzIn != null) { - this.cpuMhz = NumberTools.longToString(Long.parseLong(cpuMhzIn)); - } - else { - this.cpuMhz = null; - } - } - - /** - * @return the cpuStepping - */ - public String getCpuStepping() { - return cpuStepping; - } - - /** - * @param cpuSteppingIn the cpuStepping to set - */ - public void setCpuStepping(String cpuSteppingIn) { - this.cpuStepping = cpuSteppingIn; - } - - /** - * @return the cpuFlags - */ - public String getCpuFlags() { - return cpuFlags; - } - - /** - * @param cpuFlagsIn the cpuFlags to set - */ - public void setCpuFlags(String cpuFlagsIn) { - this.cpuFlags = cpuFlagsIn; - } - - /** - * @return the cpuModel - */ - public String getCpuModel() { - return cpuModel; - } - - /** - * @param cpuModelIn the cpuModel to set - */ - public void setCpuModel(String cpuModelIn) { - this.cpuModel = cpuModelIn; - } - - /** - * @return the cpuVersion - */ - public String getCpuVersion() { - return cpuVersion; - } - - /** - * @param cpuVersionIn the cpuVersion to set - */ - public void setCpuVersion(String cpuVersionIn) { - this.cpuVersion = cpuVersionIn; - } - - /** - * @return the cpuVendor - */ - public String getCpuVendor() { - return cpuVendor; - } - - /** - * @param cpuVendorIn the cpuVendor to set - */ - public void setCpuVendor(String cpuVendorIn) { - this.cpuVendor = cpuVendorIn; - } - - /** - * @return the cpuNrCpu - */ - public String getCpuNumberOfCpus() { - return cpuNumberOfCpus; - } - - /** - * @param cpuNumberOfCpusIn the cpuNumberOfCpus to set - */ - public void setCpuNumberOfCpus(String cpuNumberOfCpusIn) { - if (cpuNumberOfCpusIn != null) { - this.cpuNumberOfCpus = - NumberTools.longToString(Long.parseLong(cpuNumberOfCpusIn)); - } - else { - this.cpuNumberOfCpus = null; - } - } - - /** - * @return the cpuAcpiVersion - */ - public String getCpuAcpiVersion() { - return cpuAcpiVersion; - } - - /** - * @param cpuAcpiVersionIn the cpuAcpiVersion to set - */ - public void setCpuAcpiVersion(String cpuAcpiVersionIn) { - this.cpuAcpiVersion = cpuAcpiVersionIn; - } - - /** - * @return the cpuApic - */ - public String getCpuApic() { - return cpuApic; - } - - /** - * @param cpuApicIn the cpuApic to set - */ - public void setCpuApic(String cpuApicIn) { - this.cpuApic = cpuApicIn; - } - - /** - * @return the cpuApmVersion - */ - public String getCpuApmVersion() { - return cpuApmVersion; - } - - /** - * @param cpuApmVersionIn the cpuApmVersion to set - */ - public void setCpuApmVersion(String cpuApmVersionIn) { - this.cpuApmVersion = cpuApmVersionIn; - } - - /** - * @return the cpuChipset - */ - public String getCpuChipset() { - return cpuChipset; - } - - /** - * @param cpuChipsetIn the cpuChipset to set - */ - public void setCpuChipset(String cpuChipsetIn) { - this.cpuChipset = cpuChipsetIn; - } - - /** - * @return the checkin - */ - public String getCheckin() { - return checkin; - } - - /** - * @param checkinIn the checkin to set - */ - public void setCheckin(Date checkinIn) { - if (checkinIn != null) { - this.checkin = DateTools.dateToString(checkinIn, - DateTools.Resolution.MINUTE); - } - else { - this.checkin = null; - } - } - - /** - * @return the registered - */ - public String getRegistered() { - return registered; - } - - /** - * @param registeredIn the registered to set - */ - public void setRegistered(Date registeredIn) { - if (registeredIn != null) { - this.registered = DateTools.dateToString(registeredIn, - DateTools.Resolution.MINUTE); - } - else { - this.registered = null; - } - } - - /** - * @return the ram - */ - public String getRam() { - return ram; - } - - /** - * @param ramIn the ram to set - */ - public void setRam(String ramIn) { - if (ramIn != null) { - this.ram = NumberTools.longToString(Long.parseLong(ramIn)); - } - else { - this.ram = null; - } - } - - /** - * @return the swap - */ - public String getSwap() { - return swap; - } - - /** - * @param swapIn the swap to set - */ - public void setSwap(String swapIn) { - if (swapIn != null) { - this.swap = NumberTools.longToString(Long.parseLong(swapIn)); - } - else { - this.swap = null; - } - } - - /** - * @return the running kernel - */ - public String getRunningKernel() { - return runningKernel; - } - - /** - * @param runningKernelIn the runningKernel to set - */ - public void setRunningKernel(String runningKernelIn) { - if (runningKernelIn != null) { - this.runningKernel = runningKernelIn; - } - else { - this.runningKernel = null; - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java b/search-server/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java deleted file mode 100644 index 20937f9..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/models/ServerCustomInfo.java +++ /dev/null @@ -1,146 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.db.models; - - -/** - * ServerCustomInfo - * @version $Rev$ - */ -public class ServerCustomInfo extends GenericRecord { - private long keyId; - private long serverId; - private String label; - private String value; - private long createdBy; - private long lastModifiedBy; - private String created; - private String modified; - - /** - * - * @param serverIdIn server id - * @param keyIdIn key id - * @return string based on server id and key id - */ - public static String makeUniqId(long serverIdIn, long keyIdIn) { - return serverIdIn + "-" + keyIdIn; - } - - /** - * - * @return uniqId - */ - public String getUniqId() { - return makeUniqId(serverId, keyId); - } - /** - * @return the keyId - */ - public long getKeyId() { - return keyId; - } - /** - * @param keyIdIn the keyId to set - */ - public void setKeyId(long keyIdIn) { - this.keyId = keyIdIn; - } - /** - * @return the serverId - */ - public long getServerId() { - return serverId; - } - /** - * @param serverIdIn the serverId to set - */ - public void setServerId(long serverIdIn) { - this.serverId = serverIdIn; - } - /** - * @return the created - */ - public String getCreated() { - return created; - } - /** - * @param createdIn the created to set - */ - public void setCreated(String createdIn) { - this.created = createdIn; - } - /** - * @return the modified - */ - public String getModified() { - return modified; - } - /** - * @param modifiedIn the modified to set - */ - public void setModified(String modifiedIn) { - this.modified = modifiedIn; - } - /** - * @return the label - */ - public String getLabel() { - return label; - } - /** - * @param labelIn the label to set - */ - public void setLabel(String labelIn) { - this.label = labelIn; - } - /** - * @return the value - */ - public String getValue() { - return value; - } - /** - * @param valueIn the value to set - */ - public void setValue(String valueIn) { - this.value = valueIn; - } - /** - * @return the createdBy - */ - public long getCreatedBy() { - return createdBy; - } - /** - * @param createdByIn the createdBy to set - */ - public void setCreatedBy(long createdByIn) { - this.createdBy = createdByIn; - } - /** - * @return the lastModifiedBy - */ - public long getLastModifiedBy() { - return lastModifiedBy; - } - /** - * @param lastModifiedByIn the lastModifiedBy to set - */ - public void setLastModifiedBy(long lastModifiedByIn) { - this.lastModifiedBy = lastModifiedByIn; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java b/search-server/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java deleted file mode 100644 index 9ba9178..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/models/SnapshotTag.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.db.models; - - -/** - * SnapshotTag - * @version $Rev$ - */ -public class SnapshotTag extends GenericRecord { - private long id; - private long snapshotId; - private long tagNameId; - private long serverId; - private long orgId; - private String name; - private String created; - private String modified; - /** - * @return the id - */ - public long getId() { - return id; - } - /** - * @param idIn the id to set - */ - public void setId(long idIn) { - this.id = idIn; - } - /** - * @return the snapshotId - */ - public long getSnapshotId() { - return snapshotId; - } - /** - * @param snapshotIdIn the snapshotId to set - */ - public void setSnapshotId(long snapshotIdIn) { - this.snapshotId = snapshotIdIn; - } - /** - * @return the tagNameId - */ - public long getTagNameId() { - return tagNameId; - } - /** - * @param tagNameIdIn the tagNameId to set - */ - public void setTagNameId(long tagNameIdIn) { - this.tagNameId = tagNameIdIn; - } - /** - * @return the serverId - */ - public long getServerId() { - return serverId; - } - /** - * @param serverIdIn the serverId to set - */ - public void setServerId(long serverIdIn) { - this.serverId = serverIdIn; - } - /** - * @return the orgId - */ - public long getOrgId() { - return orgId; - } - /** - * @param orgIdIn the orgId to set - */ - public void setOrgId(long orgIdIn) { - this.orgId = orgIdIn; - } - /** - * @return the name - */ - public String getName() { - return name; - } - /** - * @param nameIn the name to set - */ - public void setName(String nameIn) { - this.name = nameIn; - } - /** - * @return the created - */ - public String getCreated() { - return created; - } - /** - * @param createdIn the created to set - */ - public void setCreated(String createdIn) { - this.created = createdIn; - } - /** - * @return the modified - */ - public String getModified() { - return modified; - } - /** - * @param modifiedIn the modified to set - */ - public void setModified(String modifiedIn) { - this.modified = modifiedIn; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java b/search-server/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java deleted file mode 100644 index eba617e..0000000 --- a/search-server/src/java/com/redhat/satellite/search/db/tests/DatabaseManagerTest.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.redhat.satellite.search.db.tests; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.Query; -import com.redhat.satellite.search.db.models.RhnPackage; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.List; - -public class DatabaseManagerTest extends BaseTestCase { - - private DatabaseManager dm; - - @Override - protected void setUp() throws Exception { - super.setUp(); - dm = (DatabaseManager) - container.getComponentInstance(DatabaseManager.class); - } - - public void testObjectQuery() throws IOException, SQLException { - Query<Long> maxidquery = dm.getQuery("maxPackageId"); - Long maxid = maxidquery.load(); - Query<RhnPackage> query = dm.getQuery("getPackageById"); - RhnPackage p = query.load(maxid); - assertNotNull(p); - assertEquals(p.getId(), maxid.longValue()); - } - - public void testListQuery() throws IOException, SQLException { - Query<RhnPackage> query = dm.getQuery("listPackagesFromId"); - List<RhnPackage> results = query.loadList((long) 0); - assertNotNull(results); - assertTrue(results.size() > 0); - } - - @SuppressWarnings("unchecked") - @Override - protected Class[] getComponentClasses() { - return TestUtil.buildComponentsList(DatabaseManager.class); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/DocResult.java b/search-server/src/java/com/redhat/satellite/search/index/DocResult.java deleted file mode 100644 index 2e686e7..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/DocResult.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - -import org.apache.lucene.document.Document; - - -/** - * DocResult - * @version $Rev$ - */ -public class DocResult extends Result { - - private String url = ""; - private String title = ""; - private String summary = ""; - - /** - * Constructor - */ - public DocResult() { - super(); - } - - /** - * Constructs a result object - * @param rankIn order of results returned from lucene - * @param scoreIn score of this hit as defined by lucene query - * @param doc lucene document containing data fields - */ - public DocResult(int rankIn, float scoreIn, Document doc) { - if (doc.getField("url") != null) { - setUrl(doc.getField("url").stringValue()); - setId(doc.getField("url").stringValue()); - } - if (doc.getField("title") != null) { - setTitle(doc.getField("title").stringValue()); - setName(doc.getField("title").stringValue()); - } - else { - setTitle("EMPTY"); - setName("EMPTY"); - } - setRank(rankIn); - setScore(scoreIn); - } - - /** - * @return the url - */ - public String getUrl() { - return url; - } - - /** - * @param urlIn the url to set - */ - public void setUrl(String urlIn) { - if (urlIn.startsWith("file:")) { - // Translate file:// to a usable URL - String prefix = "/rhn/help"; - int index = urlIn.indexOf(prefix); - if (index > 0) { - urlIn = urlIn.substring(index); - } - } - this.url = urlIn; - } - - /** - * @return the title - */ - public String getTitle() { - return title; - } - - /** - * @param titleIn the title to set - */ - public void setTitle(String titleIn) { - this.title = titleIn; - } - - /** - * @return the summary - */ - public String getSummary() { - return summary; - } - - /** - * @param summaryIn the summary to set - */ - public void setSummary(String summaryIn) { - this.summary = summaryIn; - } - - /** - * @return the string representation of this object - **/ - public String toString() { - return super.toString() + ", Title = " + getTitle() + ", Url = " + getUrl() + - ", Summary = " + getSummary(); - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java b/search-server/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java deleted file mode 100644 index 6bf75e9..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/HardwareDeviceResult.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - -import org.apache.lucene.document.Document; - -/** - * Result - * @version $Rev$ - */ -public class HardwareDeviceResult extends Result { - private String serverId; - - /** - * Constructor - */ - public HardwareDeviceResult() { - super(); - serverId = "N/A"; - } - - /** - * Constructs a result object - * @param rankIn order of results returned from lucene - * @param scoreIn score of this hit as defined by lucene query - * @param doc lucene document containing data fields - */ - public HardwareDeviceResult(int rankIn, float scoreIn, Document doc) { - if (doc.getField("description") != null) { - setName(doc.getField("description").stringValue()); - } - if (doc.getField("serverId") != null) { - setServerId(doc.getField("serverId").stringValue()); - } - if (doc.getField("id") != null) { - setId(doc.getField("id").stringValue()); - } - setRank(rankIn); - setScore(scoreIn); - } - - - /** - * @return the serverId - */ - public String getServerId() { - return serverId; - } - - /** - * @param serverIdIn the serverId to set - */ - public void setServerId(String serverIdIn) { - this.serverId = serverIdIn; - } - - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/IndexManager.java b/search-server/src/java/com/redhat/satellite/search/index/IndexManager.java deleted file mode 100644 index 23e4f5a..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/IndexManager.java +++ /dev/null @@ -1,837 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - -import com.redhat.satellite.search.config.Configuration; -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.ngram.NGramAnalyzer; -import com.redhat.satellite.search.index.ngram.NGramQueryParser; -import com.redhat.satellite.search.rpc.handlers.IndexHandler; - -import org.apache.hadoop.fs.FileSystem; - -import org.apache.log4j.Logger; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; -import org.apache.lucene.document.DateTools; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockObtainFailedException; - -import org.apache.nutch.analysis.AnalyzerFactory; -import org.apache.nutch.searcher.FetchedSegments; -import org.apache.nutch.searcher.HitDetails; -import org.apache.nutch.searcher.Summary; - -import org.apache.nutch.util.NutchConfiguration; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * Indexing workhorse class - * - * @version $Rev$ - */ -public class IndexManager { - - private static Logger log = Logger.getLogger(IndexManager.class); - private String indexWorkDir; - private int maxHits; - private double score_threshold; - private double system_score_threshold; - private double errata_score_threshold; - private double errata_advisory_score_threshold; - private int min_ngram; - private int max_ngram; - private boolean filterDocResults = false; - private boolean explainResults = false; - private AnalyzerFactory nutchAnalyzerFactory; - // Name conflict with our Configuration class and Hadoop's - private org.apache.hadoop.conf.Configuration nutchConf; - private Map<String, String> docLocaleLookUp = new HashMap<String, String>(); - private Map<String, FetchedSegments> docSegments; - /** - * Constructor - * - * @param config application config - */ - public IndexManager(Configuration config) { - maxHits = config.getInt("search.max_hits_returned", 50); - indexWorkDir = config.getString("search.index_work_dir", null); - if (indexWorkDir == null) { - throw new IllegalArgumentException( - "search.index_work_dir config entry " + "is missing"); - } - if (!indexWorkDir.endsWith("/")) { - indexWorkDir += "/"; - } - score_threshold = config.getDouble("search.score_threshold", .30); - system_score_threshold = config.getDouble("search.system_score_threshold", .30); - errata_score_threshold = config.getDouble("search.errata_score_threshold", .30); - errata_advisory_score_threshold = - config.getDouble("search.errata.advisory_score_threshold", .30); - min_ngram = config.getInt("search.min_ngram", 1); - max_ngram = config.getInt("search.max_ngram", 5); - initDocLocaleLookup(); - filterDocResults = config.getBoolean("search.doc.limit_results"); - explainResults = config.getBoolean("search.log.explain.results"); - initDocSummary(); - } - - - /** - * @return String of the index working directory - */ - public String getIndexWorkDir() { - return indexWorkDir; - } - - /** - * Query a index - * - * @param indexName name of the index - * @param query search query - * @param lang language - * @return list of hits - * @throws IndexingException if there is a problem indexing the content. - * @throws QueryParseException - */ - public List<Result> search(String indexName, String query, String lang) - throws IndexingException, QueryParseException { - return search(indexName, query, lang, false); - } - - /** - * Query a index - * - * @param indexName name of the index - * @param query search query - * @param lang language - * @param isFineGrained - * true: will limit results, less are returned but they are closer - * to the search query, useful for advanced/free form queries - * - * false: will allow queries to be more flexible returning words - * which are spelled similarly - * - * @return list of hits - * @throws IndexingException if there is a problem indexing the content. - * @throws QueryParseException - */ - public List<Result> search(String indexName, String query, String lang, - boolean isFineGrained) - throws IndexingException, QueryParseException { - IndexSearcher searcher = null; - IndexReader reader = null; - List<Result> retval = null; - try { - reader = getIndexReader(indexName, lang); - searcher = getIndexSearcher(indexName, lang); - QueryParser qp = getQueryParser(indexName, lang, isFineGrained); - Query q = qp.parse(query); - if (log.isDebugEnabled()) { - log.debug("Original query was: " + query); - log.debug("Parsed Query is: " + q.toString()); - } - Hits hits = searcher.search(q); - if (log.isDebugEnabled()) { - log.debug(hits.length() + " results were found."); - } - Set<Term> queryTerms = null; - try { - queryTerms = new HashSet<Term>(); - Query newQ = q.rewrite(reader); - newQ.extractTerms(queryTerms); - } - catch (Exception e) { - e.printStackTrace(); - throw new QueryParseException(e); - } - retval = processHits(indexName, hits, queryTerms, query, lang); - if (explainResults) { - debugExplainResults(indexName, hits, searcher, q, queryTerms); - } - } - catch (IOException e) { - throw new IndexingException(e); - } - catch (ParseException e) { - throw new QueryParseException("Could not parse query: '" + query + "'"); - } - finally { - try { - if (searcher != null) { - searcher.close(); - } - if (reader != null) { - reader.close(); - } - } - catch (IOException ex) { - throw new IndexingException(ex); - } - } - return retval; - } - - /** - * Adds a document to an index - * - * @param indexName index to use - * @param doc Document to be indexed. - * @param lang language. - * @throws IndexingException something went wrong adding the document - */ - public void addToIndex(String indexName, Document doc, String lang) - throws IndexingException { - - try { - IndexWriter writer = getIndexWriter(indexName, lang); - try { - writer.addDocument(doc); - writer.flush(); - } - finally { - try { - writer.close(); - } - finally { - // unlock it if it is locked. - unlockIndex(indexName); - } - } - } - catch (CorruptIndexException e) { - throw new IndexingException(e); - } - catch (LockObtainFailedException e) { - throw new IndexingException(e); - } - catch (IOException e) { - throw new IndexingException(e); - } - } - /** - * @param indexName - * @param doc document with data to index - * @param uniqueField field in doc which identifies this uniquely - * @param lang language - * @throws IndexingException - */ - public void addUniqueToIndex(String indexName, Document doc, - String uniqueField, String lang) - throws IndexingException { - IndexReader reader = null; - int numFound = 0; - try { - reader = getIndexReader(indexName, lang); - Term term = new Term(uniqueField, doc.get(uniqueField)); - numFound = reader.docFreq(term); - } - catch (FileNotFoundException e) { - // Index doesn't exist, so this add will be unique - // we don't need to do anything/ - } - catch (IOException e) { - throw new IndexingException(e); - } - finally { - if (reader != null) { - try { - reader.close(); - } - catch (IOException e) { - // - } - } - } - if (numFound > 0) { - log.info("Found " + numFound + " <" + indexName + " docs for " + - uniqueField + ":" + doc.get(uniqueField) + - " will remove them now."); - removeFromIndex(indexName, uniqueField, doc.get(uniqueField)); - } - addToIndex(indexName, doc, lang); - } - - /** - * Remove a document from an index - * - * @param indexName index to use - * @param uniqueField field name which represents this data's unique id - * @param objectId unique document id - * @throws IndexingException something went wrong removing the document - */ - public void removeFromIndex(String indexName, String uniqueField, String objectId) - throws IndexingException { - log.info("Removing <" + indexName + "> " + uniqueField + ":" + - objectId); - Term t = new Term(uniqueField, objectId); - IndexReader reader; - try { - reader = getIndexReader(indexName, IndexHandler.DEFAULT_LANG); - try { - reader.deleteDocuments(t); - reader.flush(); - } - finally { - if (reader != null) { - reader.close(); - } - } - } - catch (CorruptIndexException e) { - throw new IndexingException(e); - } - catch (IOException e) { - throw new IndexingException(e); - } - } - - /** - * Unlocks the index at the given directory if it is currently locked. - * Otherwise, does nothing. - * @param indexName index name - * @throws IOException thrown if there is a problem unlocking the index. - */ - private void unlockIndex(String indexName) throws IOException { - String path = indexWorkDir + indexName; - File f = new File(path); - Directory dir = FSDirectory.getDirectory(f); - if (IndexReader.isLocked(dir)) { - IndexReader.unlock(dir); - } - } - - private IndexWriter getIndexWriter(String name, String lang) - throws CorruptIndexException, LockObtainFailedException, - IOException { - String path = indexWorkDir + name; - File f = new File(path); - f.mkdirs(); - Analyzer analyzer = getAnalyzer(name, lang); - IndexWriter writer = new IndexWriter(path, analyzer); - writer.setUseCompoundFile(true); - return writer; - } - - private IndexReader getIndexReader(String indexName, String locale) - throws CorruptIndexException, IOException { - String path = ""; - if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { - path = indexWorkDir + File.separator + - getDocIndexPath(locale); - } - else { - path = indexWorkDir + indexName; - } - log.info("IndexManager::getIndexReader(" + indexName + ", " + locale + - ") path = " + path); - File f = new File(path); - IndexReader retval = IndexReader.open(FSDirectory.getDirectory(f)); - return retval; - } - - private IndexSearcher getIndexSearcher(String indexName, String locale) - throws CorruptIndexException, IOException { - String path = ""; - if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { - path = indexWorkDir + File.separator + - getDocIndexPath(locale); - } - else { - path = indexWorkDir + indexName; - } - log.info("IndexManager::getIndexSearcher(" + indexName + ", " + locale + - ") path = " + path); - IndexSearcher retval = new IndexSearcher(path); - return retval; - } - - private QueryParser getQueryParser(String indexName, String lang, - boolean isFineGrained) { - if (log.isDebugEnabled()) { - log.debug("getQueryParser(" + indexName + ", " + lang + ", " + - isFineGrained + ")"); - } - QueryParser qp; - Analyzer analyzer = getAnalyzer(indexName, lang); - if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { - qp = new QueryParser("content", analyzer); - } - else { - qp = new NGramQueryParser("name", analyzer, isFineGrained); - } - qp.setDateResolution(DateTools.Resolution.MINUTE); - return qp; - } - - - private Analyzer getAnalyzer(String indexName, String lang) { - if (log.isDebugEnabled()) { - log.debug("getAnalyzer(" + indexName + ", " + lang + ")"); - } - if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { - return getDocAnalyzer(lang); - } - else if (indexName.compareTo(BuilderFactory.SERVER_TYPE) == 0) { - return getServerAnalyzer(); - } - else if (indexName.compareTo(BuilderFactory.SNAPSHOT_TAG_TYPE) == 0) { - return getSnapshotTagAnalyzer(); - } - else if (indexName.compareTo(BuilderFactory.HARDWARE_DEVICE_TYPE) == 0) { - return getHardwareDeviceAnalyzer(); - } - else if (indexName.compareTo(BuilderFactory.SERVER_CUSTOM_INFO_TYPE) == 0) { - return getServerCustomInfoAnalyzer(); - } - else { - log.debug(indexName + " using getDefaultAnalyzer()"); - return getDefaultAnalyzer(); - } - } - - private List<Result> processHits(String indexName, Hits hits, Set<Term> queryTerms, - String query, String lang) - throws IOException { - List<Result> retval = new ArrayList<Result>(); - for (int x = 0; x < hits.length(); x++) { - Document doc = hits.doc(x); - Result pr = null; - if (!isScoreAcceptable(indexName, hits, x, query)) { - break; - } - if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) { - pr = new DocResult(x, hits.score(x), doc); - String summary = lookupDocSummary(doc, query, lang); - if (summary != null) { - ((DocResult)pr).setSummary(summary); - } - } - else if (indexName.compareTo(BuilderFactory.HARDWARE_DEVICE_TYPE) == 0) { - pr = new HardwareDeviceResult(x, hits.score(x), doc); - } - else if (indexName.compareTo(BuilderFactory.SNAPSHOT_TAG_TYPE) == 0) { - pr = new SnapshotTagResult(x, hits.score(x), doc); - } - else if (indexName.compareTo(BuilderFactory.SERVER_CUSTOM_INFO_TYPE) == 0) { - pr = new ServerCustomInfoResult(x, hits.score(x), doc); - } - else { - pr = new Result(x, - doc.getField("id").stringValue(), - doc.getField("name").stringValue(), - hits.score(x)); - } - if (log.isDebugEnabled()) { - log.debug("Hit[" + x + "] Score = " + hits.score(x) + ", Result = " + pr); - } - /** - * matchingField will help the webUI to understand what field was responsible - * for this match. Later implementation should use "Explanation" to determine - * field, for now we will simply grab one term and return it's field. - */ - try { - MatchingField match = new MatchingField(query, doc, queryTerms); - pr.setMatchingField(match.getFieldName()); - pr.setMatchingFieldValue(match.getFieldValue()); - log.info("hit[" + x + "] matchingField is being set to: <" + - pr.getMatchingField() + "> based on passed in query field. " + - "matchingFieldValue = " + pr.getMatchingFieldValue()); - } - catch (Exception e) { - log.error("Caught exception: ", e); - } - if (pr != null) { - retval.add(pr); - } - if (x == maxHits) { - break; - } - } - return retval; - } - /** - * - * @param indexName - * @param hits - * @param x - * @param query - * @return true - score is acceptable - * false - score is NOT acceptable - * @throws IOException - */ - private boolean isScoreAcceptable(String indexName, Hits hits, int x, String queryIn) - throws IOException { - String guessMainQueryTerm = MatchingField.getFirstFieldName(queryIn); - - if ((indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) && - (!filterDocResults)) { - return true; - } - /** - * Dropping matches which are a poor fit. - * system searches are filtered based on "system_score_threshold" - * other searches will return 10 best matches, then filter anything below - * "score_threshold" - */ - if ((indexName.compareTo(BuilderFactory.SERVER_TYPE) == 0) || - (indexName.compareTo(BuilderFactory.SERVER_CUSTOM_INFO_TYPE) == 0) || - (indexName.compareTo(BuilderFactory.SNAPSHOT_TAG_TYPE) == 0) || - (indexName.compareTo(BuilderFactory.HARDWARE_DEVICE_TYPE) == 0)) { - if (hits.score(x) < system_score_threshold) { - if (log.isDebugEnabled()) { - log.debug("hits.score(" + x + ") is " + hits.score(x)); - log.debug("Filtering out search results from " + x + " to " + - hits.length() + ", due to their score being below " + - "system_score_threshold = " + system_score_threshold); - } - return false; - } - } - else if (indexName.compareTo(BuilderFactory.ERRATA_TYPE) == 0) { - if (guessMainQueryTerm.compareTo("advisoryName") == 0) { - if (hits.score(x) < errata_advisory_score_threshold) { - if (log.isDebugEnabled()) { - log.debug("hits.score(" + x + ") is " + hits.score(x)); - log.debug("Filtering out search results from " + x + " to " + - hits.length() + ", due to their score being below " + - "errata_advisory_score_threshold = " + - errata_advisory_score_threshold); - } - return false; - } - } - else { - if (hits.score(x) < errata_score_threshold) { - if (log.isDebugEnabled()) { - log.debug("hits.score(" + x + ") is " + hits.score(x)); - log.debug("Filtering out search results from " + x + " to " + - hits.length() + ", due to their score being below " + - "errata_score_threshold = " + - errata_score_threshold); - } - return false; - } - } - } - else if (((hits.score(x) < score_threshold) && (x > 10)) || - (hits.score(x) < 0.001)) { - /** - * Dropping matches which are a poor fit. - * First term is configurable, it allows matches like spelling errors or - * suggestions to be possible. - * Second term is intended to get rid of pure and utter crap hits - */ - if (log.isDebugEnabled()) { - log.debug("hits.score(" + x + ") is " + hits.score(x)); - log.debug("Filtering out search results from " + x + " to " + - hits.length() + ", due to their score being below " + - "score_threshold = " + score_threshold); - } - return false; - } - return true; - } - - /** - * Removes any documents which are not related to the passed in Set of good value - * @param ids Set of ids of all known/good values - * @param indexName index name to operate on - * @param uniqField the name of the field in the Document to uniquely identify - * this record - * @return the number of documents deleted - */ - public int deleteRecordsNotInList(Set<String> ids, String indexName, - String uniqField) { - int count = 0; - IndexReader reader = null; - try { - reader = getIndexReader(indexName, IndexHandler.DEFAULT_LANG); - int numDocs = reader.numDocs(); - for (int i = 0; i < numDocs; i++) { - if (!reader.isDeleted(i)) { - Document doc = reader.document(i); - String uniqId = doc.getField(uniqField).stringValue(); - if (!ids.contains(uniqId)) { - log.info(indexName + ":" + uniqField + ": <" + uniqId + - "> not found in list of current/good values " + - "assuming this has been deleted from Database and we " + - "should remove it."); - removeFromIndex(indexName, uniqField, uniqId); - count++; - } - } - } - } - catch (IOException e) { - e.printStackTrace(); - log.info("deleteRecordsNotInList() caught exception : " + e); - } - catch (IndexingException e) { - e.printStackTrace(); - log.info("deleteRecordsNotInList() caught exception : " + e); - } - finally { - if (reader != null) { - try { - reader.close(); - } - catch (IOException e) { - // - } - } - } - return count; - } - - private void debugExplainResults(String indexName, Hits hits, IndexSearcher searcher, - Query q, Set<Term> queryTerms) - throws IOException { - log.debug("Parsed Query is " + q.toString()); - log.debug("Looking at index: " + indexName); - for (int i = 0; i < hits.length(); i++) { - if ((i < 10)) { - Document doc = hits.doc(i); - Float score = hits.score(i); - Explanation ex = searcher.explain(q, hits.id(i)); - log.debug("Looking at hit<" + i + ", " + hits.id(i) + ", " + score + - ">: " + doc); - log.debug("Explanation: " + ex); - MatchingField match = new MatchingField(q.toString(), doc, queryTerms); - String fieldName = match.getFieldName(); - String fieldValue = match.getFieldValue(); - log.debug("Guessing that matched fieldName is " + fieldName + " = " + - fieldValue); - } - } - } - - private String getDocIndexPath(String lang) throws IOException { - if (docLocaleLookUp.containsKey(lang)) { - return BuilderFactory.DOCS_TYPE + File.separator + - docLocaleLookUp.get(lang); - } - log.error("Unable to find docs index dir for language " + lang); - throw new IOException("Unable to find docs index dir for language: " + lang); - } - - - private Analyzer getDocAnalyzer(String lang) { - /** - * We want to use the same Analyzer nutch is using when the indexes are - * generated - * */ - Analyzer analyzer = null; - try { - analyzer = nutchAnalyzerFactory.get(lang); - } - catch (Exception e) { - log.info("Caught exception, nutch is most likely not installed"); - log.info("Defaulting to generic analyzer for Documentation Search"); - log.info("Install nutch package to get summary info and better matches."); - analyzer = new StandardAnalyzer(); - } - log.info("Language choice is " + lang + ", analyzer chosen is " + - analyzer); - return analyzer; - } - - private Analyzer getServerAnalyzer() { - PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new - NGramAnalyzer(min_ngram, max_ngram)); - analyzer.addAnalyzer("checkin", new KeywordAnalyzer()); - analyzer.addAnalyzer("registered", new KeywordAnalyzer()); - analyzer.addAnalyzer("ram", new KeywordAnalyzer()); - analyzer.addAnalyzer("swap", new KeywordAnalyzer()); - analyzer.addAnalyzer("cpuMhz", new KeywordAnalyzer()); - analyzer.addAnalyzer("cpuNumberOfCpus", new KeywordAnalyzer()); - - - return analyzer; - } - - private Analyzer getSnapshotTagAnalyzer() { - PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new - NGramAnalyzer(min_ngram, max_ngram)); - analyzer.addAnalyzer("id", new KeywordAnalyzer()); - analyzer.addAnalyzer("snapshotId", new KeywordAnalyzer()); - analyzer.addAnalyzer("orgId", new KeywordAnalyzer()); - analyzer.addAnalyzer("serverId", new KeywordAnalyzer()); - analyzer.addAnalyzer("tagNameId", new KeywordAnalyzer()); - analyzer.addAnalyzer("created", new KeywordAnalyzer()); - analyzer.addAnalyzer("modified", new KeywordAnalyzer()); - return analyzer; - } - - private Analyzer getHardwareDeviceAnalyzer() { - PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new - NGramAnalyzer(min_ngram, max_ngram)); - analyzer.addAnalyzer("id", new KeywordAnalyzer()); - analyzer.addAnalyzer("serverId", new KeywordAnalyzer()); - analyzer.addAnalyzer("pciType", new KeywordAnalyzer()); - return analyzer; - } - - private Analyzer getServerCustomInfoAnalyzer() { - PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new - NGramAnalyzer(min_ngram, max_ngram)); - analyzer.addAnalyzer("id", new KeywordAnalyzer()); - analyzer.addAnalyzer("serverId", new KeywordAnalyzer()); - analyzer.addAnalyzer("created", new KeywordAnalyzer()); - analyzer.addAnalyzer("modified", new KeywordAnalyzer()); - analyzer.addAnalyzer("createdBy", new KeywordAnalyzer()); - analyzer.addAnalyzer("lastModifiedBy", new KeywordAnalyzer()); - return analyzer; - } - - private Analyzer getDefaultAnalyzer() { - PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new - NGramAnalyzer(min_ngram, max_ngram)); - analyzer.addAnalyzer("id", new KeywordAnalyzer()); - analyzer.addAnalyzer("arch", new KeywordAnalyzer()); - analyzer.addAnalyzer("epoch", new KeywordAnalyzer()); - analyzer.addAnalyzer("version", new KeywordAnalyzer()); - analyzer.addAnalyzer("release", new KeywordAnalyzer()); - analyzer.addAnalyzer("filename", new KeywordAnalyzer()); - return analyzer; - } - - private boolean initDocSummary() { - /** - * NOTE: NutchConfiguration is expecting "nutch-default.xml" and "nutch-site.xml" - * to be available in the CLASSPATH - */ - try { - nutchConf = NutchConfiguration.create(); - nutchAnalyzerFactory = new AnalyzerFactory(nutchConf); - FileSystem fs = FileSystem.get(nutchConf); - docSegments = new HashMap<String, FetchedSegments>(); - for (String key : docLocaleLookUp.keySet()) { - String lang = docLocaleLookUp.get(key); - String segmentsDir = indexWorkDir + File.separator + - getDocIndexPath(key) + File.separator + "segments"; - FetchedSegments segments = new FetchedSegments(fs, segmentsDir, nutchConf); - if (segments == null) { - log.info("Unable to create docSegments for language: " + key); - docSegments.put(key, null); - } - String[] segNames = segments.getSegmentNames(); - if (segNames == null || segNames.length == 0) { - log.info("Unable to find any segments for language: " + key); - docSegments.put(key, null); - } - log.info("Adding Documentation segments for language: " + key); - docSegments.put(key, segments); - } - } - catch (Exception e) { - log.error("ignoring exception - most likely Nutch isn't present, so" + - " doc summaries will be empty"); - e.printStackTrace(); - } - return true; - } - - private String lookupDocSummary(Document doc, String queryString, String lang) { - if (docSegments == null) { - log.info("docSegments is null, doc summary not possible"); - log.info("nutch is probably not installed, install nutch to get summary info"); - return ""; - } - if (!docSegments.containsKey(lang)) { - log.info("Couldn't find segments info for " + lang); - log.info("Summary info will be missing for " + lang); - return ""; - } - FetchedSegments segments = docSegments.get(lang); - if (segments == null) { - log.info("Segments info for " + lang + " is null"); - return ""; - } - try { - if (log.isDebugEnabled()) { - log.debug("Attempting lookupDocSummary<" + lang + "> for " + doc); - } - HitDetails hd = new HitDetails(doc.getField("segment").stringValue(), - doc.getField("url").stringValue()); - // NOTE: Name conflict with Nutch's Query versus Lucene Query - org.apache.nutch.searcher.Query query = - org.apache.nutch.searcher.Query.parse(queryString, nutchConf); - Summary sum = segments.getSummary(hd, query); - if (log.isDebugEnabled()) { - log.debug("Will return summary<" + lang + "> = " + sum.toString()); - } - return sum.toString(); - } - catch (Exception e) { - log.info("Failed to lookupDocSummary<" + lang + ">, caught Exception: " + e); - e.printStackTrace(); - } - return ""; - } - - private void initDocLocaleLookup() { - docLocaleLookUp.put("bn_IN", "bn-IN"); - docLocaleLookUp.put("de", "de-DE"); - docLocaleLookUp.put("en_US", "en-US"); - docLocaleLookUp.put("es", "es-ES"); - docLocaleLookUp.put("fr", "fr-FR"); - docLocaleLookUp.put("gu", "gu-IN"); - docLocaleLookUp.put("hi", "hi-IN"); - docLocaleLookUp.put("it", "it-IT"); - docLocaleLookUp.put("ja", "ja-JP"); - docLocaleLookUp.put("ko", "ko-KR"); - docLocaleLookUp.put("pa", "pa-IN"); - docLocaleLookUp.put("pt_BR", "pt-BR"); - docLocaleLookUp.put("ru", "ru-RU"); - docLocaleLookUp.put("ta", "ta-IN"); - docLocaleLookUp.put("zh_CN", "zh-CN"); - docLocaleLookUp.put("zh_TW", "zh-TW"); - // Below exist in docs, but weren't available as a doc option from - // satellite webui and they weren't available Locales on my machine - // guessing at what they will look at. - docLocaleLookUp.put("as", "as-IN"); - docLocaleLookUp.put("ml", "ml-IN"); - docLocaleLookUp.put("mr", "mr-IN"); - docLocaleLookUp.put("or", "or-IN"); - docLocaleLookUp.put("kn", "kn-IN"); - docLocaleLookUp.put("si_lk", "si-LK"); - docLocaleLookUp.put("te", "te-IN"); - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/IndexingException.java b/search-server/src/java/com/redhat/satellite/search/index/IndexingException.java deleted file mode 100644 index 167de6e..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/IndexingException.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - -/** - * Something vewy vewy bad happened while indexing content - * - * @version $Rev$ - */ -public class IndexingException extends Exception { - - private static final long serialVersionUID = 3037442150973499615L; - - /** - * Simple no-arg constructor - */ - public IndexingException() { - super(); - } - - /** - * Constructor - * - * @param message - * error message - * @param cause - * root cause - */ - public IndexingException(String message, Throwable cause) { - super(message, cause); - } - - /** - * Constructor - * - * @param message - * error message - */ - public IndexingException(String message) { - super(message); - } - - /** - * Constructor - * - * @param cause - * root cause - */ - public IndexingException(Throwable cause) { - super(cause); - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java b/search-server/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java deleted file mode 100644 index 06dd221..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/KeywordAnalyzer.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index; - -import java.io.Reader; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.CharTokenizer; - - - -/** - * KeywordAnalyzer - * Used to retain all characters associated with a search term. Basically - * use this if you don't want the search term to be tokenized. - * - * @version $Rev$ - */ -public class KeywordAnalyzer extends Analyzer { - /** - * Will return the exact test passed in. - *@param fieldName ignored value - *@param reader Reader containing data to parse - *@return TokenStream which contains exact text passed in. - */ - @Override - public TokenStream tokenStream(String fieldName, Reader reader) { - return new CharTokenizer(reader) { - protected boolean isTokenChar(char c) { - return true; - } - }; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/MatchingField.java b/search-server/src/java/com/redhat/satellite/search/index/MatchingField.java deleted file mode 100644 index 8c25e88..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/MatchingField.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - -import org.apache.log4j.Logger; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.NumberTools; -import org.apache.lucene.index.Term; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.StringTokenizer; - -/** - * Purpose of this class is to guess what field in the Document is responsible - * for this being flagged as a hit. We will use the query and terms as hints. - * @version $Rev$ - */ -public class MatchingField { - private static Logger log = Logger.getLogger(IndexManager.class); - protected Document doc; - protected Object[] terms; - protected String query; - protected Map needNumberToolsAdjust; - - /** - * - * @param queryIn query used in search - * @param docIn document results returned as a match - * @param termsIn terms from a parsed query - */ - public MatchingField(String queryIn, Document docIn, Set<Term> termsIn) { - query = queryIn; - doc = docIn; - if ((termsIn != null)) { - terms = termsIn.toArray(); - } - else { - terms = new Term[0]; - } - needNumberToolsAdjust = new HashMap(); - needNumberToolsAdjust.put("cpuMhz", true); - needNumberToolsAdjust.put("cpuBogoMIPS", true); - needNumberToolsAdjust.put("cpuMhz", true); - needNumberToolsAdjust.put("cpuNumberOfCpus", true); - needNumberToolsAdjust.put("ram", true); - needNumberToolsAdjust.put("swap", true); - } - - /** - * - * @return field name most responsible for this document being a match - */ - public String getFieldName() { - if (terms.length > 0) { - return ((Term)terms[0]).field(); - } - return getFirstFieldName(query); - } - - /** - * - * @return value most responsible for this document being a match - */ - public String getFieldValue() { - String fieldName = getFieldName(); - Field f = doc.getField(fieldName); - if (f == null) { - StringBuffer sb = new StringBuffer(); - sb.append("[length=" + terms.length + "; "); - for (Object o : terms) { - sb.append(o + ", "); - } - sb.append("]"); - log.info("Unable to get matchingFieldValue for field : " + fieldName + - " with query: " + query + ", and terms = " + sb.toString()); - log.info("Document = " + doc); - return ""; - } - String value = f.stringValue(); - if (needNumberToolsAdjust.containsKey(fieldName)) { - Long temp = NumberTools.stringToLong(value); - value = temp.toString(); - } - return value; - } - - /** - * - * @param queryIn - * @return first term in query, which is a good guess as to being the most - * important term in the query. - * - * */ - public static String getFirstFieldName(String queryIn) { - // remove first parenthesis, this shows up when you do a - // Query.toString(), package searches generally have this - if (queryIn.startsWith("(")) { - queryIn = queryIn.substring(1); - } - StringTokenizer tokens = new StringTokenizer(queryIn, ":"); - return tokens.nextToken(); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/QueryParseException.java b/search-server/src/java/com/redhat/satellite/search/index/QueryParseException.java deleted file mode 100644 index e4b0744..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/QueryParseException.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index; - - -/** - * QueryParseException thrown if there is a problem parsing the query input. - * @version $Rev$ - */ -public class QueryParseException extends Exception { - /** - * Simple no-arg constructor - */ - public QueryParseException() { - super(); - } - - /** - * Constructor - * @param message error message - * @param cause root cause - */ - public QueryParseException(String message, Throwable cause) { - super(message, cause); - } - - /** - * Constructor - * @param message error message - */ - public QueryParseException(String message) { - super(message); - } - - /** - * Constructor - * @param cause root cause - */ - public QueryParseException(Throwable cause) { - super(cause); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/Result.java b/search-server/src/java/com/redhat/satellite/search/index/Result.java deleted file mode 100644 index 793c54d..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/Result.java +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - - -/** - * PackageResult - * @version $Rev$ - */ -public class Result { - private int rank = 0; - private String id = ""; - private String name = ""; - private float score = 0.0f; - - /** TODO: - * matchingField could be used to possibly give more info into what made - * lucene think this was a good match.. - */ - private String matchingField = ""; - private String matchingFieldValue = ""; - /** - * Constructor - */ - public Result() { - rank = -1; - id = ""; - name = ""; - score = 0; - } - - /** - * Constructs a pre populated - * @param r rank - * @param idIn package id - * @param nameIn package name - */ - public Result(int r, String idIn, String nameIn, float scoreIn) { - rank = r; - id = idIn; - name = nameIn; - score = scoreIn; - } - - /** - * Sets the rank - * @param rankIn rank - */ - public void setRank(int rankIn) { - rank = rankIn; - } - /** - * Returns the rank. - * @return the rank. - */ - public int getRank() { - return rank; - } - - /** - * Sets the id - * @param idIn id - */ - public void setId(String idIn) { - id = idIn; - } - - /** - * Returns the package id. - * @return the package id. - */ - public String getId() { - return id; - } - - /** - * Sets the name - * @param nameIn name - */ - public void setName(String nameIn) { - name = nameIn; - } - /** - * Returns the package name. - * @return the package name. - */ - public String getName() { - return name; - } - - /** - * Sets the score from lucene - * @param scoreIn score - */ - public void setScore(float scoreIn) { - score = scoreIn; - } - /** - * Returns the score. - * @return the score. - */ - public float getScore() { - return score; - } - - /** - * @return the matchingField - */ - public String getMatchingField() { - return matchingField; - } - - /** - * @param matchingFieldIn the matchingField to set - */ - public void setMatchingField(String matchingFieldIn) { - this.matchingField = matchingFieldIn; - } - /** - * @return the matchingFieldValue - */ - public String getMatchingFieldValue() { - return matchingFieldValue; - } - /** - * @param matchingFieldValueIn the matchingFieldValue to set - */ - public void setMatchingFieldValue(String matchingFieldValueIn) { - this.matchingFieldValue = matchingFieldValueIn; - } - /** - * @return the string representation of this object - **/ - public String toString() { - return "Name = " + getName() + ", Id = " + getId() + ", Score = " + getScore() + - ", Rank = " + getRank(); - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java b/search-server/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java deleted file mode 100644 index 06e45bd..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ServerCustomInfoResult.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - -import org.apache.lucene.document.Document; - -/** - * ServerCustomInfoResult - * @version $Rev$ - */ -public class ServerCustomInfoResult extends Result { - private String serverId; - private String value; - /** - * Constructor - */ - public ServerCustomInfoResult() { - super(); - serverId = "N/A"; - value = "N/A"; - } - - /** - * Constructs a result object - * @param rankIn order of results returned from lucene - * @param scoreIn score of this hit as defined by lucene query - * @param doc lucene document containing data fields - */ - public ServerCustomInfoResult(int rankIn, float scoreIn, Document doc) { - if (doc.getField("value") != null) { - setValue(doc.getField("value").stringValue()); - } - if (doc.getField("serverId") != null) { - setServerId(doc.getField("serverId").stringValue()); - } - if (doc.getField("id") != null) { - setId(doc.getField("id").stringValue()); - } - setRank(rankIn); - setScore(scoreIn); - } - - - /** - * @return the serverId - */ - public String getServerId() { - return serverId; - } - - /** - * @param serverIdIn the serverId to set - */ - public void setServerId(String serverIdIn) { - this.serverId = serverIdIn; - } - - /** - * @return the value - */ - public String getValue() { - return value; - } - - /** - * @param valueIn the value to set - */ - public void setValue(String valueIn) { - this.value = valueIn; - } - - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java b/search-server/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java deleted file mode 100644 index cedea5b..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/SnapshotTagResult.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index; - -import org.apache.lucene.document.Document; - -/** - * SnapshotTagResult - * @version $Rev$ - */ -public class SnapshotTagResult extends Result { - private String serverId; - private String snapshotId; - - /** - * Constructor - */ - public SnapshotTagResult() { - super(); - serverId = "N/A"; - } - - /** - * Constructs a result object - * @param rankIn order of results returned from lucene - * @param scoreIn score of this hit as defined by lucene query - * @param doc lucene document containing data fields - */ - public SnapshotTagResult(int rankIn, float scoreIn, Document doc) { - if (doc.getField("id") != null) { - setId(doc.getField("id").stringValue()); - } - if (doc.getField("name") != null) { - setName(doc.getField("name").stringValue()); - } - if (doc.getField("serverId") != null) { - setServerId(doc.getField("serverId").stringValue()); - } - if (doc.getField("snapshotId") != null) { - setSnapshotId(doc.getField("snapshotId").stringValue()); - } - setRank(rankIn); - setScore(scoreIn); - } - - - /** - * @return the serverId - */ - public String getServerId() { - return serverId; - } - - /** - * @param serverIdIn the serverId to set - */ - public void setServerId(String serverIdIn) { - this.serverId = serverIdIn; - } - - /** - * @return the snapshotId - */ - public String getSnapshotId() { - return snapshotId; - } - - /** - * @param snapshotIdIn the snapshotId to set - */ - public void setSnapshotId(String snapshotIdIn) { - this.snapshotId = snapshotIdIn; - } - - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java b/search-server/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java deleted file mode 100644 index 1717109..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/BuilderFactory.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - - -/** - * BuilderFactory - * @version $Rev$ - */ -public class BuilderFactory { - - public static final String ERRATA_TYPE = "errata"; - public static final String PACKAGES_TYPE = "packages"; - public static final String SERVER_TYPE = "server"; - public static final String DOCS_TYPE = "docs"; - public static final String HARDWARE_DEVICE_TYPE = "hwdevice"; - public static final String SNAPSHOT_TAG_TYPE = "snapshotTag"; - public static final String SERVER_CUSTOM_INFO_TYPE = "serverCustomInfo"; - /** - * Private constructor. - */ - private BuilderFactory() { - } - - /** - * Returns a Builder suitable for building the wanted type. Invalid types - * will throw an UnsupportedOperationException. - * @param type Valid type of builder. - * @return Suitable builder. - */ - public static DocumentBuilder getBuilder(String type) { - if (ERRATA_TYPE.equals(type)) { - return new ErrataDocumentBuilder(); - } - else if (PACKAGES_TYPE.equals(type)) { - return new PackageDocumentBuilder(); - } - else if (SERVER_TYPE.equals(type)) { - return new ServerDocumentBuilder(); - } - else if (HARDWARE_DEVICE_TYPE.equals(type)) { - return new HardwareDeviceDocumentBuilder(); - } - else if (SNAPSHOT_TAG_TYPE.equals(type)) { - return new SnapshotTagDocumentBuilder(); - } - else if (SERVER_CUSTOM_INFO_TYPE.equals(type)) { - return new ServerCustomInfoDocumentBuilder(); - } - else { - throw new UnsupportedOperationException(type + - " is an unsuppported type"); - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java b/search-server/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java deleted file mode 100644 index 3cb733a..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/DocumentBuilder.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - -import org.apache.lucene.document.Document; - -import java.util.Map; - - -/** - * DocumentBuilder - * @version $Rev$ - */ -public interface DocumentBuilder { - /** - * Builds a Lucene Document using the given object id and metadata map. - * The document will contain an "id" field with the value of the object id, - * and fields corresponding to the keys of the metadata and their values. - * @param objId Object id - * @param metadata key:value pairs describing the object. - * @return Lucene Document for indexing. - */ - Document buildDocument(Long objId, Map<String, String> metadata); -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java b/search-server/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java deleted file mode 100644 index 03fdd60..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/ErrataDocumentBuilder.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -import java.util.Iterator; -import java.util.Map; - - -/** - * ErrataDocumentBuilder - * @version $Rev$ - */ -public class ErrataDocumentBuilder implements DocumentBuilder { - - // TODO: add some information about fields and their indexing states. - - /** - * {@inheritDoc} - */ - public Document buildDocument(Long objId, Map<String, String> metadata) { - Document doc = new Document(); - doc.add(new Field("id", objId.toString(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); - - - for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { - Field.Store store = Field.Store.NO; - Field.Index tokenize = Field.Index.TOKENIZED; - - String name = iter.next(); - String value = metadata.get(name); - - if (name.equals("name")) { - store = Field.Store.YES; - } - else if (name.equals("advisoryName")) { - store = Field.Store.YES; - } - - doc.add(new Field(name, String.valueOf(value), store, - tokenize)); - } - return doc; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java b/search-server/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java deleted file mode 100644 index b1fd2d2..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/HardwareDeviceDocumentBuilder.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -import java.util.Iterator; -import java.util.Map; - - -/** - * HardwareDeviceDocumentBuilder - * @version $Rev$ - */ -public class HardwareDeviceDocumentBuilder implements DocumentBuilder { - - /** - * {@inheritDoc} - */ - public Document buildDocument(Long objId, Map<String, String> metadata) { - Document doc = new Document(); - doc.add(new Field("id", objId.toString(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); - - for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { - Field.Store store = Field.Store.YES; - Field.Index tokenize = Field.Index.TOKENIZED; - - String name = iter.next(); - String value = metadata.get(name); - - if (name.equals("name") || (name.equals("description"))) { - store = Field.Store.YES; - } - else if (name.equals("serverId")) { - store = Field.Store.YES; - tokenize = Field.Index.UN_TOKENIZED; - } - - doc.add(new Field(name, String.valueOf(value), store, - tokenize)); - } - return doc; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java b/search-server/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java deleted file mode 100644 index 624ad4f..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/PackageDocumentBuilder.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -import java.util.Iterator; -import java.util.Map; - - -/** - * PackageDocumentBuilder - * @version $Rev$ - */ -public class PackageDocumentBuilder implements DocumentBuilder { - - /** - * {@inheritDoc} - */ - public Document buildDocument(Long objId, Map<String, String> metadata) { - Document doc = new Document(); - doc.add(new Field("id", objId.toString(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); - - for (Iterator<String> iter = metadata.keySet().iterator(); iter - .hasNext();) { - Field.Store store = Field.Store.YES; - Field.Index tokenize = Field.Index.TOKENIZED; - - String name = iter.next(); - String value = metadata.get(name); - if (name.equals("name")) { - store = Field.Store.YES; - } - else if (name.equals("arch") || name.equals("filename") || - name.equals("version") || name.equals("release") || - name.equals("epoch")) { - tokenize = Field.Index.UN_TOKENIZED; - } - - doc.add(new Field(name, String.valueOf(value), store, - tokenize)); - } - return doc; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java b/search-server/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java deleted file mode 100644 index de10c7b..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/ServerCustomInfoDocumentBuilder.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -import java.util.Iterator; -import java.util.Map; - - -/** - * ServerCustomInfoDocumentBuilder - * @version $Rev$ - */ -public class ServerCustomInfoDocumentBuilder implements DocumentBuilder { - - /** - * {@inheritDoc} - */ - public Document buildDocument(Long objId, Map<String, String> metadata) { - Document doc = new Document(); - doc.add(new Field("id", objId.toString(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); - - for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { - Field.Store store = Field.Store.YES; - Field.Index tokenize = Field.Index.TOKENIZED; - - String name = iter.next(); - String value = metadata.get(name); - - if (name.equals("value")) { - store = Field.Store.YES; - } - else if (name.equals("created") || (name.equals("modified"))) { - store = Field.Store.YES; - tokenize = Field.Index.UN_TOKENIZED; - } - else if (name.equals("serverId") || name.equals("createdBy") || - name.equals("lastModifiedBy")) { - store = Field.Store.YES; - tokenize = Field.Index.UN_TOKENIZED; - } - - doc.add(new Field(name, String.valueOf(value), store, - tokenize)); - } - return doc; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java b/search-server/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java deleted file mode 100644 index 516c846..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/ServerDocumentBuilder.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -import java.util.Iterator; -import java.util.Map; - - -/** - * ServerDocumentBuilder - * @version $Rev$ - */ -public class ServerDocumentBuilder implements DocumentBuilder { - - /** - * {@inheritDoc} - */ - public Document buildDocument(Long objId, Map<String, String> metadata) { - Document doc = new Document(); - // Keep 'id' UN_TOKENIZED, this is needed to determine 'uniqueness' - // if you tokenize this it will break deleting documents, we'll - // no longer have unique documents per id. - - doc.add(new Field("id", objId.toString(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); - // This is the tokenized form of 'id' so we can do searches on it and - // use ngram flexibility - doc.add(new Field("system_id", objId.toString(), Field.Store.YES, - Field.Index.TOKENIZED)); - - for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { - Field.Store store = Field.Store.YES; - Field.Index tokenize = Field.Index.TOKENIZED; - - String name = iter.next(); - String value = metadata.get(name); - - if (name.equals("name") || name.equals("cpuModel") || - name.equals("hostname") || name.equals("ipaddr") || - (name.equals("runningKernel"))) { - store = Field.Store.YES; - } - else if (name.equals("checkin") || name.equals("registered") || - name.equals("ram") || name.equals("swap") || - name.equals("cpuMhz") | name.equals("cpuNumberOfCpus")) { - store = Field.Store.YES; - tokenize = Field.Index.UN_TOKENIZED; - } - - doc.add(new Field(name, String.valueOf(value), store, - tokenize)); - } - return doc; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java b/search-server/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java deleted file mode 100644 index cf055c3..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/SnapshotTagDocumentBuilder.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -import java.util.Iterator; -import java.util.Map; - - -/** - * SnapshotTagDocumentBuilder - * @version $Rev$ - */ -public class SnapshotTagDocumentBuilder implements DocumentBuilder { - - /** - * {@inheritDoc} - */ - public Document buildDocument(Long objId, Map<String, String> metadata) { - Document doc = new Document(); - doc.add(new Field("id", objId.toString(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); - - for (Iterator<String> iter = metadata.keySet().iterator(); iter.hasNext();) { - Field.Store store = Field.Store.YES; - Field.Index tokenize = Field.Index.TOKENIZED; - - String name = iter.next(); - String value = metadata.get(name); - - if (name.equals("name")) { - store = Field.Store.YES; - } - else if (name.equals("created") || (name.equals("modified"))) { - store = Field.Store.YES; - tokenize = Field.Index.UN_TOKENIZED; - } - else if (name.equals("snapshotId") || name.equals("tagNameId") || - name.equals("serverId") || name.equals("orgId")) { - store = Field.Store.YES; - tokenize = Field.Index.UN_TOKENIZED; - } - - doc.add(new Field(name, String.valueOf(value), store, - tokenize)); - } - return doc; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java b/search-server/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java deleted file mode 100644 index 530e68c..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/BuilderFactoryTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder.tests; - -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.ErrataDocumentBuilder; -import com.redhat.satellite.search.index.builder.PackageDocumentBuilder; -import com.redhat.satellite.search.index.builder.ServerDocumentBuilder; - -import junit.framework.TestCase; - - -/** - * BuilderFactoryTest - * @version $Rev$ - */ -public class BuilderFactoryTest extends TestCase { - - public void testGetBuilderErrataType() { - DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.ERRATA_TYPE); - assertNotNull(db); - assertEquals(db.getClass(), ErrataDocumentBuilder.class); - } - - public void testGetBuilderPackageType() { - DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.PACKAGES_TYPE); - assertNotNull(db); - assertEquals(db.getClass(), PackageDocumentBuilder.class); - } - - public void testGetBuilderServerType() { - DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.SERVER_TYPE); - assertNotNull(db); - assertEquals(db.getClass(), ServerDocumentBuilder.class); - } - - public void testGetBuilderInvalid() { - try { - BuilderFactory.getBuilder(null); - fail("getBuilder should have thrown exception with null type"); - } - catch(UnsupportedOperationException uoe) { - assertTrue(true); - } - - try { - BuilderFactory.getBuilder("foobar"); - fail("getBuilder should have thrown exception with null type"); - } - catch(UnsupportedOperationException uoe) { - assertTrue(true); - } - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java b/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java deleted file mode 100644 index a4dcfc6..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ErrataDocumentBuilderTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder.tests; - -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.ErrataDocumentBuilder; - -import org.apache.lucene.document.Document; - -import java.util.HashMap; -import java.util.Map; - -import junit.framework.TestCase; - - -/** - * ErrataDocumentBuilderTest - * @version $Rev$ - */ -public class ErrataDocumentBuilderTest extends TestCase { - - public void testBuildDocument() { - Map<String,String> metadata = new HashMap<String,String>(); - metadata.put("id", new Long(10).toString()); - metadata.put("advisory", "Advisory"); - metadata.put("advisoryType", "AdvisoryType"); - metadata.put("advisoryName", "AdvisoryName"); - metadata.put("advisoryRel", new Long(100).toString()); - metadata.put("product", "Product"); - metadata.put("description", "Description"); - metadata.put("synopsis", "Synopsis"); - metadata.put("topic", "Topic"); - metadata.put("solution", "Solution"); - metadata.put("issueDate", "IssueDate"); - metadata.put("updateDate", "UpdateDate"); - metadata.put("notes", "Notes"); - metadata.put("orgId", "OrgId"); - metadata.put("created", "Created"); - metadata.put("modified", "Modified"); - metadata.put("lastModified", "LastModified"); - metadata.put("severityId", new Long(1).toString()); - metadata.put("name", "Advisory"); - - DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.ERRATA_TYPE); - assertTrue(db instanceof ErrataDocumentBuilder); - Document doc = db.buildDocument(new Long(10), metadata); - - assertNotNull(doc); - assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); - assertEquals(doc.getField("advisory").stringValue(), "Advisory"); - assertEquals(doc.getField("advisoryType").stringValue(), "AdvisoryType"); - assertEquals(doc.getField("advisoryName").stringValue(), "AdvisoryName"); - assertEquals(doc.getField("advisoryRel").stringValue(), new Long(100).toString()); - assertEquals(doc.getField("product").stringValue(), "Product"); - assertEquals(doc.getField("description").stringValue(), "Description"); - assertEquals(doc.getField("synopsis").stringValue(), "Synopsis"); - assertEquals(doc.getField("topic").stringValue(), "Topic"); - assertEquals(doc.getField("solution").stringValue(), "Solution"); - assertEquals(doc.getField("issueDate").stringValue(), "IssueDate"); - assertEquals(doc.getField("updateDate").stringValue(), "UpdateDate"); - assertEquals(doc.getField("notes").stringValue(), "Notes"); - assertEquals(doc.getField("orgId").stringValue(), "OrgId"); - assertEquals(doc.getField("created").stringValue(), "Created"); - assertEquals(doc.getField("modified").stringValue(), "Modified"); - assertEquals(doc.getField("lastModified").stringValue(), "LastModified"); - assertEquals(doc.getField("severityId").stringValue(), new Long(1).toString()); - assertEquals(doc.getField("name").stringValue(), "Advisory"); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java b/search-server/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java deleted file mode 100644 index 452a171..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/HardwareDeviceDocumentBuilderTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder.tests; - -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.HardwareDeviceDocumentBuilder; - -import org.apache.lucene.document.Document; - -import junit.framework.TestCase; -import java.util.HashMap; -import java.util.Map; - - - -/** - * HardwareDeviceDocumentBuilderTest - * @version $Rev$ - */ -public class HardwareDeviceDocumentBuilderTest extends TestCase { - - public void testBuilderDocument() { - Map<String, String> metadata = new HashMap<String, String>(); - metadata.put("name", "Name"); - metadata.put("description", "Description"); - - DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.HARDWARE_DEVICE_TYPE); - assertTrue(db instanceof HardwareDeviceDocumentBuilder); - Document doc = db.buildDocument(new Long(10), metadata); - - assertNotNull(doc); - assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); - assertEquals(doc.getField("name").stringValue(), "Name"); - assertEquals(doc.getField("description").stringValue(), "Description"); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java b/search-server/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java deleted file mode 100644 index 2d77640..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/PackageDocumentBuilderTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder.tests; - -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.PackageDocumentBuilder; - -import org.apache.lucene.document.Document; - -import java.util.HashMap; -import java.util.Map; - -import junit.framework.TestCase; - - -/** - * PackageDocumentBuilderTest - * @version $Rev$ - */ -public class PackageDocumentBuilderTest extends TestCase { - - public void testBuildDocument() { - Map<String, String> metadata = new HashMap<String, String>(); - metadata.put("name", "Name"); - metadata.put("version", "PrettyVersion"); - metadata.put("filename", "FileName"); - metadata.put("description", "Description"); - metadata.put("summary", "Summary"); - metadata.put("arch", "Arch"); - - DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.PACKAGES_TYPE); - assertTrue(db instanceof PackageDocumentBuilder); - Document doc = db.buildDocument(new Long(10), metadata); - - assertNotNull(doc); - assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); - assertEquals(doc.getField("name").stringValue(), "Name"); - assertEquals(doc.getField("version").stringValue(), "PrettyVersion"); - assertEquals(doc.getField("filename").stringValue(), "FileName"); - assertEquals(doc.getField("description").stringValue(), "Description"); - assertEquals(doc.getField("summary").stringValue(), "Summary"); - assertEquals(doc.getField("arch").stringValue(), "Arch"); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java b/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java deleted file mode 100644 index 89935a3..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerCustomInfoDocumentBuilderTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder.tests; - -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.ServerCustomInfoDocumentBuilder; - -import org.apache.lucene.document.Document; - -import junit.framework.TestCase; -import java.util.HashMap; -import java.util.Map; - - - -/** - * ServerCustomInfoDocumentBuilderTest - * @version $Rev$ - */ -public class ServerCustomInfoDocumentBuilderTest extends TestCase { - - public void testBuilderDocument() { - Map<String, String> metadata = new HashMap<String, String>(); - metadata.put("value", "value"); - metadata.put("serverId", "server id"); - - DocumentBuilder db = BuilderFactory.getBuilder( - BuilderFactory.SERVER_CUSTOM_INFO_TYPE); - assertTrue(db instanceof ServerCustomInfoDocumentBuilder); - Document doc = db.buildDocument(new Long(10), metadata); - - assertNotNull(doc); - assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); - assertEquals(doc.getField("value").stringValue(), "value"); - assertEquals(doc.getField("serverId").stringValue(), "server id"); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java b/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java deleted file mode 100644 index c1ee3e3..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/ServerDocumentBuilderTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder.tests; - -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.ServerDocumentBuilder; - -import org.apache.lucene.document.Document; - -import junit.framework.TestCase; -import java.util.HashMap; -import java.util.Map; - - - -/** - * ServerDocumentBuilderTest - * @version $Rev$ - */ -public class ServerDocumentBuilderTest extends TestCase { - - public void testBuilderDocument() { - Map<String, String> metadata = new HashMap<String, String>(); - metadata.put("name", "Name"); - metadata.put("description", "Description"); - metadata.put("info", "Info"); - - DocumentBuilder db = BuilderFactory.getBuilder(BuilderFactory.SERVER_TYPE); - assertTrue(db instanceof ServerDocumentBuilder); - Document doc = db.buildDocument(new Long(10), metadata); - - assertNotNull(doc); - assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); - assertEquals(doc.getField("name").stringValue(), "Name"); - assertEquals(doc.getField("description").stringValue(), "Description"); - assertEquals(doc.getField("info").stringValue(), "Info"); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java b/search-server/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java deleted file mode 100644 index 6b0fe32..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/builder/tests/SnapshotTagDocumentBuilderTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.builder.tests; - -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.SnapshotTagDocumentBuilder; - -import org.apache.lucene.document.Document; - -import junit.framework.TestCase; -import java.util.HashMap; -import java.util.Map; - - - -/** - * SnapshotTagDocumentBuilderTest - * @version $Rev$ - */ -public class SnapshotTagDocumentBuilderTest extends TestCase { - - public void testBuilderDocument() { - Map<String, String> metadata = new HashMap<String, String>(); - metadata.put("name", "Name"); - - DocumentBuilder db = - BuilderFactory.getBuilder(BuilderFactory.SNAPSHOT_TAG_TYPE); - assertTrue(db instanceof SnapshotTagDocumentBuilder); - Document doc = db.buildDocument(new Long(10), metadata); - - assertNotNull(doc); - assertEquals(doc.getField("id").stringValue(), new Long(10).toString()); - assertEquals(doc.getField("name").stringValue(), "Name"); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java b/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java deleted file mode 100644 index fb39ca0..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramAnalyzer.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.index.ngram; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LowerCaseFilter; -import org.apache.lucene.analysis.standard.StandardTokenizer; -import org.apache.lucene.analysis.standard.StandardFilter; -import org.apache.lucene.analysis.TokenStream; -// From Lucene Sandbox -import org.apache.lucene.analysis.ngram.NGramTokenFilter; - -import java.io.Reader; - -/** - * NGramAnalyzer - * A ngram will take a term and break it up into a series of smaller - * permutations of different letter combinations. - * @version $Rev$ - */ -public class NGramAnalyzer extends Analyzer { - //Controls minimum size ngram to construct - protected int min_ngram; - //Controls maximum size ngram to construct - protected int max_ngram; - - /** - * Constructor - * @param min min length of ngram to generate - * @param max max length of ngram to generate - */ - public NGramAnalyzer(int min, int max) { - super(); - min_ngram = min; - max_ngram = max; - } - /** - * Constructs a pre populated - * @param reader contains data to parse - * @return TokenStream of ngrams - */ - public TokenStream tokenStream(Reader reader) { - return tokenStream(null, reader); - } - - /** - * @param fieldName ignored param - * @param reader contains data to parse - * @return TokenStream of ngrams - */ - public TokenStream tokenStream(String fieldName, Reader reader) { - return new NGramTokenFilter( - new LowerCaseFilter( - new StandardFilter( - new StandardTokenizer(reader))), min_ngram, max_ngram); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java b/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java deleted file mode 100644 index 8d14250..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQuery.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - - -package com.redhat.satellite.search.index.ngram; - -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.index.Term; - -import java.io.IOException; -import java.io.StringReader; - -/** - * NGramQuery - * A custom BooleanQuery, it takes each ngram-token and adds as an OR term. - * @version $Rev$ - */ -public class NGramQuery extends BooleanQuery { - - private static final long serialVersionUID = 1L; - - /** - * Constructor - * @param field name of the field - * @param queryTerms String containing a term or a series of terms to search. - * The string will be parsed and will be broken up into a series of NGrams. - * @throws IOException something went wrong parsing queryTerms - * */ - public NGramQuery(String field, String queryTerms, int min, int max) - throws IOException { - NGramAnalyzer nga = new NGramAnalyzer(min, max); - TokenStream ngrams = nga.tokenStream(new StringReader(queryTerms)); - Token token; - while ((token = ngrams.next()) != null) { - Term t = new Term(field, new String(token.termBuffer()).trim()); - add(new TermQuery(t), BooleanClause.Occur.SHOULD); - } - } - - /** - * - * @param pq PhraseQuery to break up and convert to NGramQuery - * Forms a BooleanQuery with each term in the original PhraseQuery OR'd. - * Note: Assumes that each term has already been tokenized into a ngram, - * this method will not re-tokenize terms. - * @param useMust controls if BooleanClause.Occur SHOULD or MUST is used. - */ - public NGramQuery(PhraseQuery pq, boolean useMust) { - Term[] terms = pq.getTerms(); - for (int i = 0; i < terms.length; i++) { - BooleanClause.Occur occur = BooleanClause.Occur.SHOULD; - if (useMust) { - occur = BooleanClause.Occur.MUST; - } - add(new TermQuery(terms[i]), occur); - } - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java b/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java deleted file mode 100644 index 7ae2311..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ngram/NGramQueryParser.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.ngram; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.lucene.search.ConstantScoreRangeQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.DateTools; -import org.apache.lucene.document.NumberTools; - -import org.apache.log4j.Logger; - -/** - * NGramQueryParser - * Creates a custom query parser of ngram-tokenized search terms - * - * @version $Rev$ - */ -public class NGramQueryParser extends QueryParser { - - private static Logger log = Logger.getLogger(NGramQueryParser.class); - private boolean useMust = false; - /** - * Constructor - * @param f field name - * @param a analyzer - * @param useMustIn boolean option set to true when doing a Free Form or - * advanced search, it will force each NGramQuery to be constructed using - * the BooleanClause.Occur.MUST. This narrows returned search results and - * is not recommended for regular searches which want spelling variances - * to be tolerated. - */ - public NGramQueryParser(String f, Analyzer a, boolean useMustIn) { - super(f, a); - this.setDateResolution(DateTools.Resolution.DAY); - this.useMust = useMustIn; - } - - /** - * Constructor - * @param f field name - * @param a analyzer - */ - public NGramQueryParser(String f, Analyzer a) { - super(f, a); - this.setDateResolution(DateTools.Resolution.DAY); - } - - /** - * - * @return value for useMust - */ - public boolean getUseMust() { - return useMust; - } - - /** - * - * @param value for useMust - */ - public void setUseMust(boolean value) { - useMust = value; - } - - protected Query getFieldQuery(String defaultField, - String queryText) throws ParseException { - Query orig = super.getFieldQuery(defaultField, queryText); - if (!(orig instanceof PhraseQuery)) { - log.debug("Returning default query. No phrase query translation."); - return orig; - } - /** - * A ngram when parsed will become a series of smaller search terms, - * these terms are grouped together into a PhraseQuery. We are taking - * that PhraseQuery and breaking out each ngram term then combining all - * ngrams together to form a BooleanQuery. - */ - PhraseQuery pq = (PhraseQuery)orig; - return new NGramQuery(pq, useMust); - } - - /** - * - * @param field - * @return return true if this looks to be a date string - * false if this is not a date string - */ - protected boolean isDate(String field) { - if (field.length() == 12) { - return true; - } - return false; - } - /** - * This will look to see if "part1" or "part2" are strings of all digits, - * if they are, then they will be converted to a lexicographically safe string - * representation, then passed into the inherited getRangeQuery(). This is needed when - * comparing something like "4" to be less than "10". - * If the strings don't fit the pattern of all digits, then they get passed through - * to the inherited getRangeQuery(). - */ - protected Query getRangeQuery(String field, - String part1, - String part2, - boolean inclusive) throws ParseException { - if (isDate(part1) && isDate(part2)) { - if (log.isDebugEnabled()) { - log.debug("Detected passed in terms are dates, creating " + - "ConstantScoreRangeQuery(" + field + ", " + part1 + ", " + - part2 + ", " + inclusive + ", " + inclusive); - } - return new ConstantScoreRangeQuery(field, part1, part2, inclusive, - inclusive); - } - String newPart1 = part1; - String newPart2 = part2; - String regEx = "(\d)*"; - Pattern pattern = Pattern.compile(regEx); - Matcher matcher1 = pattern.matcher(part1); - Matcher matcher2 = pattern.matcher(part2); - if (matcher1.matches() && matcher2.matches()) { - newPart1 = NumberTools.longToString(Long.parseLong(part1)); - newPart2 = NumberTools.longToString(Long.parseLong(part2)); - if (log.isDebugEnabled()) { - log.debug("NGramQueryParser.getRangeQuery() Converted " + part1 + " to " + - newPart1 + ", Converted " + part2 + " to " + newPart2); - } - } - return super.getRangeQuery(field, newPart1, newPart2, inclusive); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java b/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java deleted file mode 100644 index 0ed65f3..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramAnalyzerTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.ngram.tests; - -import com.redhat.satellite.search.index.ngram.NGramAnalyzer; - -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenStream; - -import org.apache.log4j.Logger; - -import java.io.StringReader; - -public class NGramAnalyzerTest extends NGramTestSetup { - - private static Logger log = Logger.getLogger(NGramAnalyzerTest.class); - - public NGramAnalyzerTest() { - super(); - } - - public void testTokenStream() throws Exception { - NGramAnalyzer nga = new NGramAnalyzer(min_ngram, max_ngram); - TokenStream ngrams = nga.tokenStream(new StringReader("aspell")); - Token token; - String result = new String(""); - while ((token = ngrams.next()) != null) { - result += new String(token.termBuffer()).trim() + ","; - } - log.info("Created a ngram token stream, this is what it looks like: " - + result); - - assertTrue("testTokenStream", result.compareTo("a,s,p,e,l,l,as,sp,pe," + - "el,ll,asp,spe,pel,ell,aspe,spel,pell,aspel,spell,") == 0); - - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java b/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java deleted file mode 100644 index e5077bb..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryParserTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.ngram.tests; - -import java.io.IOException; - -import com.redhat.satellite.search.index.ngram.NGramAnalyzer; -import com.redhat.satellite.search.index.ngram.NGramQueryParser; - -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; - -public class NGramQueryParserTest extends NGramTestSetup { - - private static Logger log = Logger.getLogger(NGramQueryParserTest.class); - - public NGramQueryParserTest() { - super(); - } - - public void testBasicQueryParse() throws Exception { - String defaultField = new String("name"); - NGramQueryParser parser = new NGramQueryParser(defaultField, new NGramAnalyzer(min_ngram, max_ngram)); - String txt = new String("spell"); - Query q = parser.parse(txt); - log.info("testBasicQueryParse() query = " + q.toString()); - assertTrue(q.toString().compareTo("name:s name:p name:e name:l name:l " + - "name:sp name:pe name:el name:ll name:spe name:pel name:ell " + - "name:spel name:pell name:spell") == 0); - } - - /** - * We want to make sure that when searching for multiple terms, each term will become it's - * own BooleanQuery. - * */ - public void testMultiPhraseQueryParse() throws Exception { - String defaultField = new String("name"); - NGramQueryParser parser = new NGramQueryParser(defaultField, new NGramAnalyzer(min_ngram, max_ngram)); - String txt = new String("spell* virt manager"); - Query q = parser.parse(txt); - assertTrue(q instanceof BooleanQuery); - BooleanQuery bq = (BooleanQuery)q; - assertTrue(bq.getClauses().length == 3); - } - - public void testWildcardQueryParse() throws Exception { - NGramQueryParser parser = new NGramQueryParser("name", new NGramAnalyzer(min_ngram, max_ngram)); - String txt = new String("spell*"); - Query q = parser.parse(txt); - log.info("Wildcard query = " + q.toString()); - assertTrue(q.toString().compareTo("name:spell*") == 0); - } - - public void testQueryParseWithSpecialChars() throws Exception { - String queryString = new String("spell* virt- manager+"); - log.info("testQueryParserWithSpecialChars(): query string is: " + queryString); - NGramQueryParser parser = new NGramQueryParser("name", new NGramAnalyzer(min_ngram, max_ngram)); - Query q = parser.parse(queryString); - log.info("Using NGramQueryParser query = " + q.toString()); - - QueryParser origParser = new QueryParser("name", new StandardAnalyzer()); - q = origParser.parse(queryString); - log.info("Using QueryParser query = " + q.toString()); - } - - public Hits performSearch(Directory dir, String query, boolean useMust) - throws Exception { - - NGramQueryParser parser = new NGramQueryParser("name", - new NGramAnalyzer(min_ngram, max_ngram), useMust); - IndexSearcher searcher = new IndexSearcher(dir); - Query q = parser.parse(query); - Hits hits = searcher.search(q); - log.info("Original Query = " + query); - log.info("Parsed Query = " + q.toString()); - log.info("Hits.length() = " + hits.length()); - for (int i=0; i < hits.length(); i++) { - log.debug("Document<"+hits.id(i)+"> = " + hits.doc(i)); - //Explanation explain = searcher.explain(q, hits.id(i)); - //log.debug("explain = " + explain.toString()); - } - return hits; - } - - public void testFreeFormQueryParse() throws Exception { - String queryString = new String("name:spell -description:another"); - log.info("Original query: " + queryString); - - NGramQueryParser parser = new NGramQueryParser("name", - new NGramAnalyzer(min_ngram, max_ngram), true); - Query q = parser.parse(queryString); - log.info("NGramQueryParser parsed query: " + q.toString()); - - QueryParser origParser = new QueryParser("name", new StandardAnalyzer()); - q = origParser.parse(queryString); - log.info("QueryParser parsed query = " + q.toString()); - } - - - - public void testFreeFormSearch() throws Exception { - Hits hits = null; - String query = null; - boolean useMust = true; - // Grab all packages with name "spell" AND - // description does NOT contain "another" - query = "name:spell -description:another"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(hits.length() == 2); - - // Grab all packages with name "virt" AND - // description MUST have "factory" in it - query = "name:virt +description:factory"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(hits.length() == 2); - - // Grab all packages with name "virt" - query = "name:virt description:factory"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(hits.length() == 4); - - query = "name:virt OR description:factory"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(hits.length() == 4); - - query = "name:virt AND description:factory"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(hits.length() == 1); - - query = "name:virt -description:factory"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(hits.length() == 2); - } - /** - * - * */ - public void testBasicSearch() throws Exception { - Hits hits; - String query; - boolean useMust = false; - query = "spell"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(thresholdHits(hits) == 5); - assertTrue(hits.length() == 16); - - query = "aspelll"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(thresholdHits(hits) == 4); - assertTrue(hits.length() == 17); - - query = "aspell"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(thresholdHits(hits) == 4); - assertTrue(hits.length() == 17); - - query = "pel"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(thresholdHits(hits) == 8); - assertTrue(hits.length() == 16); - - query = "gtk"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(thresholdHits(hits) == 7); - assertTrue(hits.length() == 17); - - - // We want a search for kernel-hugemem to return kernel-hugemem as top hit - // but currently, kernel-hugemem-devel is matchin instead. This test - // is a placeholder as we explore ways to fix this. - query = "((name:kernel-hugemem)^2 (description:kernel-hugemem) " + - "(filename:kernel-hugemem))"; - hits = performSearch(this.ngramDir, query, useMust); - displayHits(hits); - assertTrue(thresholdHits(hits) == 3); - assertTrue(hits.length() == 20); - String firstHitName = hits.doc(0).get("name"); - assertTrue(firstHitName.compareToIgnoreCase("kernel-hugemem-devel") == 0); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java b/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java deleted file mode 100644 index d8dadb2..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramQueryTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.ngram.tests; - -import com.redhat.satellite.search.index.ngram.NGramQuery; - -import org.apache.lucene.search.Query; - -import org.apache.log4j.Logger; - -public class NGramQueryTest extends NGramTestSetup { - - private static Logger log = Logger.getLogger(NGramQueryTest.class); - - public NGramQueryTest() { - super(); - } - - public void testCreateQuery() throws Exception { - String term = "spell"; - Query q = new NGramQuery("name", term, min_ngram, max_ngram); - log.info("NGramQuery("+term+") = " + q.toString()); - assertTrue(q != null); - } - - public void testCreateQueryMinMax() throws Exception { - String term = "spell"; - Query q = new NGramQuery("name", term, min_ngram, max_ngram); - log.info("NGramQuery("+term+") = " + q.toString()); - assertTrue(q.toString().compareTo("name:s name:p name:e name:l " + - "name:l name:sp name:pe name:el name:ll name:spe name:pel " + - "name:ell name:spel name:pell name:spell") == 0); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java b/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java deleted file mode 100644 index d3ac974..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/ngram/tests/NGramTestSetup.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.index.ngram.tests; - -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - - -import com.redhat.satellite.search.index.ngram.NGramAnalyzer; - -import org.apache.log4j.Logger; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; - -import junit.framework.TestCase; - - -public class NGramTestSetup extends TestCase { - private static Logger log = Logger.getLogger(NGramTestSetup.class); - - protected RAMDirectory ngramDir; - protected RAMDirectory stanDir; - - protected double score_threshold = .10; - protected int min_ngram = 1; - protected int max_ngram = 5; - - protected List<Map<String,String>> items = - new LinkedList<Map<String, String>>(); - - public NGramTestSetup() { - super(); - } - - protected void addItem(String name, String description, String filename) { - Map<String, String> item = new HashMap<String, String>(); - item.put("name", name); - item.put("description", description); - item.put("filename", filename); - items.add(item); - } - - protected void initItems() { - addItem("spellingbee", "spelling application", "spellingbee-1.0.rpm"); - addItem("aspelling", "another spelling program alternate spell", - "aspelling-1.0.rpm"); - addItem("aspell", "another spelling program", "aspell-0.3.4.rpm"); - addItem("spell", "spelling program", "spell-4943.rpm"); - addItem("slelp", "application with spelling error", "slelp1-43.rpm"); - addItem("libvirt", "virtualization library", "virt-pkg-1.rpm"); - addItem("virt-manager", "blah blah application", "virt-pkg-1.rpm"); - addItem("virtualfactory", "virtual factory something", - "virtfact-04.rpm"); - addItem("newFactory", "factory test application", "newFactory-1.9.rpm"); - addItem("gtk+-devel", "development library for gtk", - "gtk+-devel-10.rpm"); - addItem("gtk+", "runtime library", "gtk+-30.rpm"); - addItem("gtk-doc", "documentation for gtk", "gtk-doc-393.rpm"); - addItem("authconfig-gtk", "authentication related gtk", - "authconfig-gtk-039.rpm"); - addItem("mtr-gtk", "blah blah mtr gtk", "mtr-gtk-039.rpm"); - addItem("ghostscript-gtk", "printting support application gtk", - "ghostscript-gtk-30.rpm"); - addItem("gnome-bluetooth-libs", "library for bluetooth support", - "gnome-bluetooth-libs-3.4.rpm"); - addItem("scim-bridge-gtk", "blah blah scim gtk", - "scim-bridge-gtk-494.rpm"); - addItem("kernel", "linux kernel package", "kernel-094.rpm"); - addItem("kernel-hugemem", "This package includes an SMP version of " + - "the Linux kernel which supports systems with 16 Gigabytes " + - "of memory or more.", "kernel-hugemem-2.6.9-84.EL.i686"); - addItem("kernel-hugemem-devel", "This package provides kernel " + - "headers +and makefiles sufficient to build modules against " + - "the hugemem kernel package.", - "kernel-hugemem-devel-2.6.9-84.EL.i686"); - } - /** - * Creates an index in RAM - * */ - public void setUp() throws Exception { - super.setUp(); - initItems(); - this.stanDir = new RAMDirectory(); - IndexWriter stanWriter = new IndexWriter(this.stanDir, new StandardAnalyzer(), true); - - this.ngramDir = new RAMDirectory(); - IndexWriter ngramWriter = new IndexWriter(this.ngramDir, new NGramAnalyzer(min_ngram, max_ngram), true); - - for (Map<String, String> item: items) { - String name = item.get("name"); - String descp = item.get("description"); - Document doc = new Document(); - doc.add(new Field("name", name, Field.Store.YES, Field.Index.TOKENIZED)); - doc.add(new Field("description", descp, Field.Store.YES, - Field.Index.TOKENIZED)); - stanWriter.addDocument(doc); - ngramWriter.addDocument(doc); - } - stanWriter.close(); - ngramWriter.close(); - } - - public Hits performSearch(Directory dir, Analyzer alyz, String query) throws Exception { - QueryParser parser = new QueryParser("name", alyz); - IndexSearcher searcher = new IndexSearcher(dir); - Query q = parser.parse(query); - Hits hits = searcher.search(q); - return hits; - } - - protected int thresholdHits(Hits hits) throws IOException { - /** We could consider doing thresholding as a relative thing... - * instead of checking against an absolute value, we grab top score - * then filter based on difference from that... - */ - int counter = 0; - for (int i=0; i < hits.length(); i++) { - if (hits.score(i) >= score_threshold) { - counter++; - } - else { - break; - } - } - return counter; - } - - protected void displayHits(Hits hits) throws IOException { - for (int i = 0; i < hits.length(); i++) { - Document doc = hits.doc(i); - String name = doc.get("name"); - String description = doc.get("description"); - log.info("Hit<" + i + "> Score< " + hits.score(i) + "> name = <" + - name + "> description = <" + description + ">"); - } - } - - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java b/search-server/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java deleted file mode 100644 index 238073f..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/tests/IndexManagerTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.redhat.satellite.search.index.tests; - -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.index.Result; -import com.redhat.satellite.search.index.QueryParseException; -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; -import com.redhat.satellite.search.index.builder.PackageDocumentBuilder; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; -import com.redhat.satellite.search.config.Configuration; - -import org.apache.lucene.document.Document; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class IndexManagerTest extends BaseTestCase { - - private IndexManager indexManager; - - public void setUp() throws Exception { - super.setUp(); - indexManager = (IndexManager) - container.getComponentInstance(IndexManager.class); - } - - public void testIndexing() throws IOException, IndexingException { - String index = "foo"; - Long objectId = new Long(123); - Map<String, String> meta = new HashMap<String, String>(); - meta.put("name", "foo"); - meta.put("desc", "A really nice foo"); - meta.put("size", "12345"); - meta.put("dateCreated", "7/13/2007"); - DocumentBuilder pdb = new PackageDocumentBuilder(); - Document doc = pdb.buildDocument(objectId, meta); - indexManager.addToIndex(index, doc, "en"); - } - - public void testQuerying() - throws IOException, IndexingException, QueryParseException { - - String index = "foo"; - Long objectId = new Long(123); - Map<String, String> meta = new HashMap<String, String>(); - meta.put("name", "foo"); - meta.put("desc", "A really nice foo"); - meta.put("size", "12345"); - meta.put("dateCreated", "7/13/2007"); - DocumentBuilder pdb = new PackageDocumentBuilder(); - Document doc = pdb.buildDocument(objectId, meta); - indexManager.addToIndex(index, doc, "en"); - List<Result> results = indexManager.search(index, "name:foo", "en"); - assertTrue(results.size() >= 1); - results = indexManager.search(index, "desc:really", "en"); - assertTrue(results.size() >= 1); - } - - - public void StillNeedsWork_testQueryDocs() - throws IOException, IndexingException, QueryParseException { - - Configuration config = TestUtil.makeConfiguration(); - config.setString("search.index_work_dir", "/usr/share/rhn/search/indexes/"); - IndexManager indexMgr = new IndexManager(config); - // - // TODO: Revisit how docs data gets injected for testing. - // Currently relying on docs data to already exist for now. - // - System.out.println("We are expecting nutch to have been run previously, " + - "and the index files to be copied to "+ config.getString("search.index_work_dir") + - BuilderFactory.DOCS_TYPE); - - String index = BuilderFactory.DOCS_TYPE; - String query = new String("redhat"); - List<Result> results = indexMgr.search(index, query, "en"); - System.out.println("Number of results returned is " + results.size()); - assertTrue(results.size() >= 1); - - } - - @SuppressWarnings("unchecked") - @Override - protected Class[] getComponentClasses() { - return TestUtil.buildComponentsList(IndexManager.class); - } - - -} diff --git a/search-server/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java b/search-server/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java deleted file mode 100644 index fa1c516..0000000 --- a/search-server/src/java/com/redhat/satellite/search/index/tests/KeywordAnalyzerTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.redhat.satellite.search.index.tests; - -import com.redhat.satellite.search.index.KeywordAnalyzer; - -import org.apache.lucene.analysis.TokenStream; - -import junit.framework.TestCase; - -import java.io.StringReader; - -public class KeywordAnalyzerTest extends TestCase { - - public void processString(String originalValue) throws Exception { - KeywordAnalyzer ka = new KeywordAnalyzer(); - StringReader sr = new StringReader(originalValue); - TokenStream ts = ka.tokenStream("ignoredField", sr); - assertTrue("Text Should be Untouched", new String(ts.next().termBuffer()).trim(). - compareTo(originalValue) == 0); - assertTrue("Token should be null", ts.next() == null); - } - public void testBasicParse() throws Exception { - processString("i386"); - processString("bx-gh-3&^0-993$#@!%^&*()-=+_><?/.,';:[]}{)"); - processString("j839,. 43 ..,.-=-=`1~!@#"); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/RpcServer.java b/search-server/src/java/com/redhat/satellite/search/rpc/RpcServer.java deleted file mode 100644 index 695c301..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/RpcServer.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc; - -import com.redhat.satellite.search.config.Configuration; -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.scheduler.ScheduleManager; - -import org.picocontainer.Startable; - -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.SocketAddress; -import java.net.UnknownHostException; -import java.util.Iterator; -import java.util.Map; - -import redstone.xmlrpc.XmlRpcServer; -import simple.http.connect.Connection; -import simple.http.connect.ConnectionFactory; - -/** - * Manages the embedded webserver and configures the XML-RPC runtime - * - * @version $Rev$ - */ -public class RpcServer implements Startable { - - private int listenPort; - private InetAddress listenAddress; - private Map<String, String> handlers; - private XmlRpcServer xmlrpcServer; - private ServerSocket socket; - private IndexManager indexManager; - private DatabaseManager databaseManager; - private ScheduleManager scheduleManager; - - /** - * Constructor - * - * @param config - * dependency - * @param idxManager - * dependency - * @throws UnknownHostException - * if config contains bad rpc_address - */ - public RpcServer(Configuration config, IndexManager idxManager, - DatabaseManager dbManager, ScheduleManager schedMgr) throws - UnknownHostException { - listenPort = config.getInt("search.rpc_port", 2828); - String addr = config.getString("search.rpc_address", "127.0.0.1"); - if (addr != null) { - listenAddress = InetAddress.getByName(addr); - } - handlers = config.getMap("search.rpc_handlers"); - if (handlers == null) { - throw new IllegalArgumentException( - "search.rpc_handlers config entry " + "is missing"); - } - indexManager = idxManager; - databaseManager = dbManager; - scheduleManager = schedMgr; - } - - /** - * Starts the server - * @throws RuntimeException something bad happened - */ - public void start() { - xmlrpcServer = new XmlRpcServer(); - for (Iterator<String> iter = handlers.keySet().iterator(); iter - .hasNext();) { - String name = iter.next(); - String className = handlers.get(name); - xmlrpcServer.addInvocationHandler(name, loadHandler(className)); - } - XmlRpcInvoker invoker = new XmlRpcInvoker(xmlrpcServer); - Connection connection = ConnectionFactory.getConnection(invoker); - try { - socket = new ServerSocket(); - SocketAddress sockAddr = null; - if (listenAddress != null) { - sockAddr = new InetSocketAddress(listenAddress, listenPort); - } - else { - sockAddr = new InetSocketAddress(listenPort); - } - socket.bind(sockAddr); - connection.connect(socket); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - - - /** - * Stops the server - * - * @throws IOException - */ - public void stop() { - try { - socket.close(); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - @SuppressWarnings("unchecked") - private Object loadHandler(String className) { - try { - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - Class klass = cl.loadClass(className); - Class[] paramTypes = { IndexManager.class, DatabaseManager.class, - ScheduleManager.class }; - Object[] params = { indexManager, databaseManager, scheduleManager}; - Constructor c = klass.getConstructor(paramTypes); - return c.newInstance(params); - } - catch (NoSuchMethodException e) { - throw new RuntimeException(e); - } - catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - catch (IllegalArgumentException e) { - throw new RuntimeException(e); - } - catch (InstantiationException e) { - throw new RuntimeException(e); - } - catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - catch (InvocationTargetException e) { - throw new RuntimeException(e); - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java b/search-server/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java deleted file mode 100644 index 0352d05..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/XmlRpcInvoker.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc; - -import org.apache.log4j.Logger; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintStream; -import java.io.StringWriter; - -import redstone.xmlrpc.XmlRpcServer; -import simple.http.ProtocolHandler; -import simple.http.Request; -import simple.http.Response; - -/** - * SimpleWeb ProtocolHandler used to pass requests to the XML-RPC layer - * - * @version $Rev$ - * - */ -public class XmlRpcInvoker implements ProtocolHandler { - - private static Logger log = Logger.getLogger(XmlRpcInvoker.class); - private XmlRpcServer server; - - /** - * Constructor - * - * @param xmlrpcServer - * handles actual XML-RPC calls - */ - public XmlRpcInvoker(XmlRpcServer xmlrpcServer) { - server = xmlrpcServer; - } - - /** - * {@inheritDoc} - */ - public void handle(Request request, Response response) { - String uri = request.getURI(); - log.info(uri); - try { - if (!uri.startsWith("/RPC2")) { - response.setCode(404); - response.setText(uri); - PrintStream out = response.getPrintStream(); - out.println("<html><body><title>Page not found</title>"); - out.println("<b>" + uri + " not found</b>"); - out.println("</body></html>"); - response.set("Content-Type", "text/html"); - out.flush(); - out.close(); - } - else { - InputStream in = request.getInputStream(); - StringWriter writer = new StringWriter(); - server.execute(in, writer); - OutputStream out = response.getOutputStream(); - response.set("Content-Type", "text/xml"); - writer.flush(); - out.write(writer.getBuffer().toString().getBytes()); - out.flush(); - out.close(); - } - } - catch (IOException e) { - e.printStackTrace(); - } - finally { - try { - response.commit(); - } - catch (IOException e) { - e.printStackTrace(); - } - } - - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java b/search-server/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java deleted file mode 100644 index 3222978..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/AdminHandler.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc.handlers; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.scheduler.ScheduleManager; - -import org.apache.log4j.Logger; - -/** - * XML-RPC handler which handles calls for administration - * Updating indexes maybe more tasks later - * - * @version $Rev$ - */ -public class AdminHandler { - - private static Logger log = Logger.getLogger(AdminHandler.class); - private ScheduleManager scheduleManager; - - /** - * Constructor - * - * @param idxManager - * Search engine interface - */ - public AdminHandler(IndexManager idxManager, DatabaseManager dbMgr, - ScheduleManager schedMgr) { - log.info("** AdminHandler constructor invoked"); - scheduleManager = schedMgr; - } - /** - * Causes the task associated with the indexName to run and index new data. - * - * @param indexName - * @return true if index update is scheduled, false if unable to schedule. - */ - public boolean updateIndex(String indexName) { - if (log.isDebugEnabled()) { - log.debug("AdminHandler::updateIndex(" + indexName); - } - return scheduleManager.triggerIndexTask(indexName); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java b/search-server/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java deleted file mode 100644 index 85e9872..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/DatabaseHandler.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc.handlers; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.Query; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.Result; -import com.redhat.satellite.search.index.QueryParseException; -import com.redhat.satellite.search.scheduler.ScheduleManager; - -import com.ibatis.sqlmap.client.SqlMapException; - -import org.apache.log4j.Logger; - -import redstone.xmlrpc.XmlRpcFault; - -import java.sql.SQLException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * XML-RPC handler which handles calls for database queries - * - * TODO: - * 1) Restrict this class to only being able to call queries - * defined in specified namespaces, for now that would be: "errataHandler", - * errata_handler.xml. - * 2) Screenhits returned, verify these objects are allowed to be seen by the logged in - * user. - * - * - * @version $Rev$ - */ -public class DatabaseHandler { - - private static Logger log = Logger.getLogger(DatabaseHandler.class); - private DatabaseManager databaseManager; - /** - * Constructor - * @param idxManager IndexManager - unused param, needed for interface compatibility - * with RpcServer constructing us. - * @param dbMgr DatabaseManager - * Will perform database queries - * @param schedManager ScheduleManager unused param, needed for interface compatibility - * with RpcServer constructing us. - */ - public DatabaseHandler(IndexManager idxManager, DatabaseManager dbMgr, - ScheduleManager schedManager) { - databaseManager = dbMgr; - } - - /** - * Search database - * - * @param sessionId - * user's application session id - * @param namespace - * namespace query is located in - * @param query - * search query - * @return list of document ids as results - * @throws XmlRpcFault something bad happened - */ - public List<Result> search(long sessionId, String namespace, String query) - throws XmlRpcFault { - if (log.isDebugEnabled()) { - log.debug("DatabaseHandler:: searching for <" + namespace + ">: " + query); - } - try { - String queryName = getQueryName(query); - Map<String, String> params = getQueryParams(query); - if (log.isDebugEnabled()) { - log.debug("Calling runQuery(" + sessionId + ", " + namespace + - ", " + queryName + ", " + params + ")"); - } - List<Result> hits = runQuery(sessionId, namespace, queryName, - params); - log.info("Returned " + hits.size() + " records"); - return hits; - } - catch (SqlMapException e) { - log.warn("Caught SqlMapException: " + e.getMessage()); - e.printStackTrace(); - throw new XmlRpcFault(IndexHandler.DB_ERROR, e.getMessage()); - } - catch (SQLException e) { - log.warn("Caught SQLException: " + e.getMessage()); - e.printStackTrace(); - throw new XmlRpcFault(IndexHandler.DB_ERROR, e.getMessage()); - } - catch (QueryParseException e) { - log.warn("Caught QueryParseException: " + e.getMessage()); - e.printStackTrace(); - throw new XmlRpcFault(IndexHandler.QUERY_ERROR, e.getMessage()); - } - } - - /** - * - * @param queryIn format "name(param1, param2, ....)" - * @return query name from query - * @throws QueryParseException thrown when queryIn is malformed - */ - private String getQueryName(String queryIn) throws QueryParseException { - queryIn = queryIn.trim(); - int index = queryIn.indexOf(":("); - if (index < 0) { - throw new QueryParseException("Could not parse query: '" + queryIn + "'"); - } - String queryName = queryIn.substring(0, index); - return queryName; - } - /** - * - * @param queryIn String format of "name(param1, param2, ....)" - * @return List of params from query - * @throws QueryParseException thrown when queryIn is malformed - */ - private Map<String, String> getQueryParams(String queryIn) throws QueryParseException { - queryIn = queryIn.trim(); - String delim = ":("; - int index = queryIn.indexOf(delim); - if (index < 0) { - throw new QueryParseException("Could not parse query: '" + queryIn + "'"); - } - int indexB = queryIn.indexOf(")", index); - if (indexB < 0) { - throw new QueryParseException("Could not parse query: '" + queryIn + "'"); - } - index = index + delim.length() - 1; // We want to be pointed to end of location of - // delim - String[] temps = queryIn.substring(index + delim.length() - 1, indexB).split(","); - Map params = new HashMap<String, String>(); - for (int i = 0; i < temps.length; i++) { - params.put("param" + new Integer(i).toString(), temps[i].trim()); - } - return params; - } - - /** Will need to generalize this... right now it's specific to Errata only - * - * @param namespace namespace query lives in - * @param queryName query to run - * @param args parameters to the query - * @return List of results - */ - private List<Result> runQuery(Long sessionId, String namespace, String queryName, - Map<String, String> args) - throws SQLException, SqlMapException { - // Look up Query in DB - args.put("sessionId", sessionId.toString()); - Query query = databaseManager.getQuery(queryName); - List<Result> retval; - try { - retval = query.loadList(args); - } - finally { - query.close(); - } - return retval; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java b/search-server/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java deleted file mode 100644 index 0c972e0..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/IndexHandler.java +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc.handlers; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.Query; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.index.Result; -import com.redhat.satellite.search.index.QueryParseException; -import com.redhat.satellite.search.scheduler.ScheduleManager; - -import org.apache.log4j.Logger; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -import redstone.xmlrpc.XmlRpcFault; - -/** - * XML-RPC handler which handles calls for indexing - * - * @version $Rev$ - */ -public class IndexHandler { - - private static Logger log = Logger.getLogger(IndexHandler.class); - private IndexManager indexManager; - private DatabaseManager databaseManager; - public static final int QUERY_ERROR = 100; - public static final int INDEX_ERROR = 200; - public static final int DB_ERROR = 300; - public static final String DEFAULT_LANG = new Locale("EN", "US").toString(); - - /** - * Constructor - * - * @param idxManager - * Search engine interface - */ - public IndexHandler(IndexManager idxManager, DatabaseManager dbMgr, - ScheduleManager schedMgr) { - indexManager = idxManager; - databaseManager = dbMgr; - } - - /** - * Search index - - * assumes English language as default language - * - * @param sessionId - * user's application session id - * @param indexName - * index to use - * @param query - * search query - * @return list of document ids as results - * @throws XmlRpcFault something bad happened - */ - public List<Result> search(long sessionId, String indexName, String query) - throws XmlRpcFault { - return search(sessionId, indexName, query, DEFAULT_LANG); - } - - /** - * Search index - - * assumes English language as default language - * - * @param sessionId - * user's application session id - * @param indexName - * index to use - * @param query - * search query - * @param isFineGrained - * if set will restrict matches to be stricter and less forgiving - * @return list of document ids as results - * @throws XmlRpcFault something bad happened - */ - public List<Result> search(long sessionId, String indexName, String query, - boolean isFineGrained) - throws XmlRpcFault { - return search(sessionId, indexName, query, DEFAULT_LANG, isFineGrained); - } - - /** - * Search index - * - * @param sessionId - * user's application session id - * @param indexName - * index to use - * @param query - * search query - * @param lang - * language - * @return list of document ids as results - * @throws XmlRpcFault something bad happened - */ - public List<Result> search(long sessionId, String indexName, String query, - String lang) throws XmlRpcFault { - return search(sessionId, indexName, query, lang, false); - } - - /** - * Search index - * - * @param sessionId - * user's application session id - * @param indexName - * index to use - * @param query - * search query - * @param lang - * language - * @param isFineGrained - * if set will restrict matches to be stricter and less forgiving - * @return list of document ids as results - * @throws XmlRpcFault something bad happened - */ - public List<Result> search(long sessionId, String indexName, String query, - String lang, boolean isFineGrained) - throws XmlRpcFault { - if (log.isDebugEnabled()) { - log.debug("IndexHandler:: searching for: " + query + ", indexName = " + - indexName + ", lang = " + lang); - } - try { - List<Result> hits = indexManager.search(indexName, query, lang, - isFineGrained); - if (indexName.compareTo("package") == 0) { - return screenHits(sessionId, indexName, hits); - } - return hits; - } - catch (IndexingException e) { - log.error("Caught exception: ", e); - throw new XmlRpcFault(INDEX_ERROR, e.getMessage()); - } - catch (QueryParseException e) { - log.error("Caught exception: ", e); - throw new XmlRpcFault(QUERY_ERROR, e.getMessage()); - } - catch (SQLException e) { - log.error("Caught exception: ", e); - throw new XmlRpcFault(DB_ERROR, e.getMessage()); - } - } - - private List<Result> screenHits(long sessionId, String indexName, - List<Result> hits) throws SQLException { - - if (hits == null || hits.size() == 0) { - if (log.isDebugEnabled()) { - log.debug("NO HITS FOUND"); - } - return Collections.EMPTY_LIST; - } - List<String> ids = new ArrayList<String>(); - for (Result pr : hits) { - ids.add(pr.getId()); - } - - Query<String> query = null; - if ("package".equals(indexName)) { - query = databaseManager.getQuery("verifyPackageVisibility"); - } - else if ("errata".equals(indexName)) { - query = databaseManager.getQuery("verifyErrataVisibility"); - } - else { - if (log.isDebugEnabled()) { - log.debug("screenHits(" + indexName + - ") no 'screening of results' performed"); - log.debug("results: " + hits); - } - return hits; - } - - Map params = new HashMap(); - params.put("session_id", sessionId); - params.put("id_list", ids); - - try { - List<String> visible = query.loadList(params); - if (log.isDebugEnabled()) { - log.debug("results: " + visible); - } - - // add the PackageResults that match the visible list - List<Result> realResults = new ArrayList<Result>(); - for (Result pr : hits) { - if (visible.contains(pr.getId())) { - realResults.add(pr); - } - } - return realResults; - } - finally { - query.close(); - } - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java b/search-server/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java deleted file mode 100644 index 1b88713..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/VisibilityResultsHandler.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc.handlers; - -import com.redhat.satellite.search.db.ResultHandler; - -import org.apache.log4j.Logger; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -/** - * Cheesy class for screening hit results based - * on user's security context - * - * @version $Rev $ - */ -class VisibilityResultsHandler implements ResultHandler { - private static Logger log = Logger - .getLogger(VisibilityResultsHandler.class); - private List<String> results = new ArrayList<String>(); - - - /** - * {@inheritDoc} - */ - public void handleRow(ResultSet rs) throws SQLException { - long id = rs.getLong(1); - results.add(String.valueOf(id)); - } - - /** - * Getter for accumulated results - * @return results - */ - List<String> getResults() { - log.warn("returning results"); - return results; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java b/search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java deleted file mode 100644 index 15150b7..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/DatabaseHandlerTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc.handlers.tests; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.rpc.handlers.DatabaseHandler; -import com.redhat.satellite.search.scheduler.ScheduleManager; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; - -import org.apache.log4j.Logger; - -import java.util.List; - -import redstone.xmlrpc.XmlRpcFault; - - -public class DatabaseHandlerTest extends BaseTestCase { - - private static Logger log = Logger.getLogger(DatabaseHandlerTest.class); - - @Override - @SuppressWarnings("unchecked") - protected Class[] getComponentClasses() { - Class[] components = {IndexManager.class, - DatabaseManager.class, - ScheduleManager.class}; - - return TestUtil.buildComponentsList(components); - } - - public void testQuery() throws XmlRpcFault, InterruptedException { - DatabaseManager db = (DatabaseManager) - container.getComponentInstance(DatabaseManager.class); - IndexManager idx = (IndexManager) - container.getComponentInstance(IndexManager.class); - ScheduleManager schedMgr = (ScheduleManager) - container.getComponentInstance(ScheduleManager.class); - DatabaseHandler handler = new DatabaseHandler(idx, db, schedMgr); - String searchNamespace = "errata"; - String searchString = "listErrataByIssueDateRange:(2004-10-20, 2010-10-30)"; - log.info("Calling hander.search(1, " + searchNamespace + ", " + searchString + ")"); - List results = handler.search(1, searchNamespace, searchString); - log.info("results = " + results); - assertTrue(results.size() > 0); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java b/search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java deleted file mode 100644 index 6f69a95..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/handlers/tests/IndexHandlerTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.rpc.handlers.tests; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.rpc.handlers.IndexHandler; -import com.redhat.satellite.search.scheduler.ScheduleManager; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; - -import org.apache.log4j.Logger; - -import java.util.List; - -import redstone.xmlrpc.XmlRpcFault; - -public class IndexHandlerTest extends BaseTestCase { - - private static Logger log = Logger.getLogger(IndexHandlerTest.class); - - @Override - @SuppressWarnings("unchecked") - protected Class[] getComponentClasses() { - Class[] components = {IndexManager.class, - DatabaseManager.class, - ScheduleManager.class}; - - return TestUtil.buildComponentsList(components); - } - - public void testQuery() throws XmlRpcFault, InterruptedException { - // Let the indexing task do some stuff - Thread.sleep(15000); - DatabaseManager db = (DatabaseManager) - container.getComponentInstance(DatabaseManager.class); - IndexManager idx = (IndexManager) - container.getComponentInstance(IndexManager.class); - ScheduleManager schedMgr = (ScheduleManager) - container.getComponentInstance(ScheduleManager.class); - IndexHandler handler = new IndexHandler(idx, db, schedMgr); - handler.search(252437, "package", "description:package", "en"); - List results = handler.search(252437, "package", "kernel*", "en"); - log.info("kernel results 1: " + results); - results = handler.search(252437, "package", "kernel", "en"); - log.info("kernel results 2: " + results); - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java b/search-server/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java deleted file mode 100644 index 4a1282e..0000000 --- a/search-server/src/java/com/redhat/satellite/search/rpc/tests/RpcServerTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.redhat.satellite.search.rpc.tests; - -import com.redhat.satellite.search.rpc.RpcServer; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; - -public class RpcServerTest extends BaseTestCase { - @SuppressWarnings("unchecked") - @Override - protected Class[] getComponentClasses() { - return TestUtil.buildComponentsList(RpcServer.class); - } - - public void testStopServer() { - RpcServer server = (RpcServer) - container.getComponentInstanceOfType(RpcServer.class); - server.stop(); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java b/search-server/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java deleted file mode 100644 index ce19352..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/ScheduleManager.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.scheduler; - -import com.redhat.satellite.search.config.Configuration; -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.scheduler.tasks.IndexErrataTask; -import com.redhat.satellite.search.scheduler.tasks.IndexPackagesTask; -import com.redhat.satellite.search.scheduler.tasks.IndexSnapshotTagsTask; -import com.redhat.satellite.search.scheduler.tasks.IndexServerCustomInfoTask; -import com.redhat.satellite.search.scheduler.tasks.IndexSystemsTask; -import com.redhat.satellite.search.scheduler.tasks.IndexHardwareDevicesTask; - -import org.apache.log4j.Logger; - -import org.picocontainer.Startable; -import org.quartz.JobDataMap; -import org.quartz.JobDetail; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.SimpleTrigger; -import org.quartz.Trigger; -import org.quartz.impl.StdSchedulerFactory; - -import java.util.Date; - -/** - * Manages all scheduled tasks for the search server - * Right now all tasks are hardcoded -- this should be - * changed to be more config file driven - * - * @version $Rev $ - */ -public class ScheduleManager implements Startable { - private static Logger log = Logger.getLogger(ScheduleManager.class); - private Scheduler scheduler; - private DatabaseManager databaseManager; - private IndexManager indexManager; - - private final String updateIndexGroupName = "updateIndex"; - /** - * Constructor - * @param dbmgr allows ScheduleManager to access the database. - * @param idxmgr allows the ScheduleManager to access the indexer. - */ - public ScheduleManager(DatabaseManager dbmgr, IndexManager idxmgr) { - databaseManager = dbmgr; - indexManager = idxmgr; - try { - scheduler = StdSchedulerFactory.getDefaultScheduler(); - } - catch (SchedulerException e) { - throw new RuntimeException(e); - } - } - - private void scheduleJob(Scheduler sched, String name, - int mode, long interval, Class task, JobDataMap data) - throws SchedulerException { - - Trigger t = createTrigger(name, updateIndexGroupName, mode, interval); - JobDetail d = new JobDetail(name, updateIndexGroupName, task); - d.setJobDataMap(data); - sched.scheduleJob(d, t); - } - - private Trigger createTrigger(String name, String group, int mode, - long interval) { - Trigger trigger = new SimpleTrigger(name, "default", name, group, - new Date(), null, mode, interval); - trigger.setMisfireInstruction( - SimpleTrigger.MISFIRE_INSTRUCTION_RESCHEDULE_NEXT_WITH_EXISTING_COUNT); - return trigger; - } - - /** - * {@inheritDoc} - */ - public void start() { - try { - Configuration config = new Configuration(); - - long interval = config.getInt("search.schedule.interval", 300000); - log.info("ScheduleManager task interval is set to " + interval); - int mode = SimpleTrigger.REPEAT_INDEFINITELY; - if (System.getProperties().get("isTesting") != null) { - interval = 100; - mode = 0; - } - - JobDataMap jobData = new JobDataMap(); - jobData.put("indexManager", indexManager); - jobData.put("databaseManager", databaseManager); - jobData.put("configuration", new Configuration()); - - scheduleJob(scheduler, BuilderFactory.PACKAGES_TYPE, - mode, interval, - IndexPackagesTask.class, jobData); - - scheduleJob(scheduler, BuilderFactory.ERRATA_TYPE, - mode, interval, - IndexErrataTask.class, jobData); - - scheduleJob(scheduler, BuilderFactory.SERVER_TYPE, - mode, interval, - IndexSystemsTask.class, jobData); - - scheduleJob(scheduler, BuilderFactory.HARDWARE_DEVICE_TYPE, - mode, interval, - IndexHardwareDevicesTask.class, jobData); - - scheduleJob(scheduler, BuilderFactory.SNAPSHOT_TAG_TYPE, - mode, interval, - IndexSnapshotTagsTask.class, jobData); - - scheduleJob(scheduler, BuilderFactory.SERVER_CUSTOM_INFO_TYPE, - mode, interval, - IndexServerCustomInfoTask.class, jobData); - - scheduler.start(); - } - catch (SchedulerException e) { - throw new RuntimeException(e); - } - } - - /** - * {@inheritDoc} - */ - public void stop() { - try { - scheduler.shutdown(); - } - catch (SchedulerException e) { - throw new RuntimeException(e); - } - } - - private boolean isSupported(String indexName) { - if (BuilderFactory.ERRATA_TYPE.equals(indexName) || - BuilderFactory.HARDWARE_DEVICE_TYPE.equals(indexName) || - BuilderFactory.PACKAGES_TYPE.equals(indexName) || - BuilderFactory.SERVER_CUSTOM_INFO_TYPE.equals(indexName) || - BuilderFactory.SERVER_TYPE.equals(indexName) || - BuilderFactory.SNAPSHOT_TAG_TYPE.equals(indexName)) { - return true; - } - else if (BuilderFactory.DOCS_TYPE.equals(indexName)) { - log.info("Index updates for " + BuilderFactory.DOCS_TYPE + - " are not supported."); - return false; - } - log.info("Unknown index: " + indexName); - return false; - } - - /** - * Will create/schedule a trigger for the passed in indexName. - * Note: Only one trigger per indexName is allowed, if subsequent calls - * are made before the current trigger finishes completion, this request - * will be dropped. - * @param indexName - * @return - */ - public boolean triggerIndexTask(String indexName) { - if (!isSupported(indexName)) { - log.info(indexName + " is not a supported for scheduler modifications."); - return false; - } - // Define a Trigger that will fire "now" and associate it with the existing job - Trigger trigger = new SimpleTrigger("immediateTrigger-" + indexName, - "group1", new Date()); - trigger.setJobName(indexName); - trigger.setJobGroup(updateIndexGroupName); - try { - // Schedule the trigger - log.info("Scheduling trigger: " + trigger); - scheduler.scheduleJob(trigger); - } - catch (SchedulerException e) { - log.warn("Scheduling trigger: " + trigger + " failed."); - log.warn("Exception was caught: ", e); - return false; - } - return true; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java deleted file mode 100644 index 48e7b94..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/GenericIndexTask.java +++ /dev/null @@ -1,286 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks; - -import com.ibatis.sqlmap.client.SqlMapException; -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.Query; -import com.redhat.satellite.search.db.WriteQuery; -import com.redhat.satellite.search.db.models.GenericRecord; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; - -import org.apache.log4j.Logger; -import org.apache.lucene.document.Document; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -import java.sql.SQLException; - -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * GenericIndexTask - * @version $Rev$ - */ -public abstract class GenericIndexTask implements Job { - - private static Logger log = Logger.getLogger(GenericIndexTask.class); - private String lang = "en"; - /** - * {@inheritDoc} - */ - public void execute(JobExecutionContext ctx) - throws JobExecutionException { - JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); - DatabaseManager databaseManager = - (DatabaseManager)jobData.get("databaseManager"); - IndexManager indexManager = - (IndexManager)jobData.get("indexManager"); - - try { - List<GenericRecord> data = getRecords(databaseManager); - int count = 0; - log.info(super.getClass().toString() + "found [" + - data.size() + "] items to index"); - for (Iterator<GenericRecord> iter = data.iterator(); iter.hasNext();) { - GenericRecord current = iter.next(); - indexRecord(indexManager, current); - count++; - if (count == 10 || !iter.hasNext()) { - if (System.getProperties().get("isTesting") == null) { - updateLastRecord(databaseManager, current.getId()); - } - count = 0; - } - } - // - // Check to see if any records have been deleted from database, so - // we should delete from our indexes. - // - int numDel = handleDeletedRecords(databaseManager, indexManager); - log.info("Deleted " + numDel + " records from index <" + - getIndexName() + ">"); - } - catch (SQLException e) { - throw new JobExecutionException(e); - } - catch (IndexingException e) { - throw new JobExecutionException(e); - } - } - /** - * @param databaseManager - * @param sid - */ - private void updateLastRecord(DatabaseManager databaseManager, long sid) - throws SQLException { - - WriteQuery updateQuery = databaseManager.getWriterQuery(getQueryUpdateLastRecord()); - WriteQuery insertQuery = databaseManager.getWriterQuery(getQueryCreateLastRecord()); - - try { - Map params = new HashMap(); - params.put("id", sid); - params.put("last_modified", Calendar.getInstance().getTime()); - - if (updateQuery.update(params) == 0) { - insertQuery.insert(params); - } - } - finally { - try { - if (updateQuery != null) { - updateQuery.close(); - } - } - finally { - if (insertQuery != null) { - insertQuery.close(); - } - } - } - } - - /** - * @param indexManager - * @param current - */ - private void indexRecord(IndexManager indexManager, - GenericRecord data) - throws IndexingException { - - Map<String, String> attrs = getFieldMap(data); - log.info(super.getClass().toString() + " Indexing object: " + - data.getId() + ": " + attrs.toString()); - DocumentBuilder pdb = BuilderFactory.getBuilder(getIndexName()); - Document doc = pdb.buildDocument(new Long(data.getId()), attrs); - indexManager.addUniqueToIndex(getIndexName(), doc, getUniqueFieldId(), - lang); - } - - - /** - * @param databaseManager - * @return - */ - private List<GenericRecord> getRecords(DatabaseManager databaseManager) - throws SQLException { - // What was the last object id we indexed? - List<GenericRecord> retval = null; - Query<Long> query = databaseManager.getQuery(getQueryLastRecord()); - Long sid = null; - try { - sid = query.load(); - } - finally { - query.close(); - } - if (sid == null) { - sid = new Long(0); - } - // When was the last time we ran the indexing of servers? - Query<Date> queryLast = databaseManager.getQuery(getQueryLastIndexDate()); - Date lastRun = null; - try { - lastRun = queryLast.load(); - } - finally { - queryLast.close(); - } - if (lastRun == null) { - lastRun = new Date(0); - } - // Lookup what objects have not been indexed, or need to be reindexed. - Query<GenericRecord> srvrQuery = databaseManager.getQuery( - getQueryRecordsToIndex()); - try { - Map params = new HashMap(); - params.put("id", sid); - params.put("last_modified", lastRun); - log.info("GenericIndexTask<" + super.getClass().toString() + - " last processed id = " + sid + ", lastRun was " + lastRun); - retval = srvrQuery.loadList(params); - log.info("GenericIndexTask<" + super.getClass().toString() + - " number of results returned = " + retval.size()); - } - finally { - srvrQuery.close(); - } - return retval; - } - - /** - * Will determine if any records have been deleted from the DB, then will - * delete those records from the lucene index. - * @return number of deleted records - */ - protected int handleDeletedRecords(DatabaseManager databaseManager, - IndexManager indexManager) - throws SQLException { - List<Long> ids = null; - Query<Long> query = null; - String uniqField = null; - String indexName = null; - HashSet<String> idSet = null; - try { - query = databaseManager.getQuery(getQueryAllIds()); - ids = query.loadList(Collections.EMPTY_MAP); - if ((ids == null) || (ids.size() == 0)) { - log.info("Got back no data from '" + getQueryAllIds() + "'"); - log.info("Skipping the handleDeletedRecords() method"); - return 0; - } - idSet = new HashSet(); - for (Long num : ids) { - idSet.add(num.toString()); - } - uniqField = getUniqueFieldId(); - indexName = getIndexName(); - } - catch (SqlMapException e) { - e.printStackTrace(); - log.info("Error with 'getQueryAllIds()' on " + - super.getClass().toString()); - //just print the warning so we know and skip this method. - return 0; - } - finally { - if (query != null) { - query.close(); - } - } - return indexManager.deleteRecordsNotInList(idSet, indexName, uniqField); - } - - /** - * - * @param data fully populated DTO object - * @return map which represents the fields to index along with their values - * @throws ClassCastException when data isn't castable to the intended DTO - */ - protected abstract Map<String, String> getFieldMap(GenericRecord data) - throws ClassCastException; - /** - * - * @return the index name - */ - public abstract String getIndexName(); - /** - * @return the Document field name which represents the unique id for this data - */ - public abstract String getUniqueFieldId(); - /** - * - * @return name of query which shows the last record indexed - */ - protected abstract String getQueryLastRecord(); - /** - * - * @return name of query which will update the last record indexed - */ - protected abstract String getQueryUpdateLastRecord(); - /** - * - * @return name of query which will create the last record indexed - */ - protected abstract String getQueryCreateLastRecord(); - /** - * - * @return name of query which will give back records to be indexed - */ - protected abstract String getQueryRecordsToIndex(); - /** - * - * @return name of query which will show the date this task last ran - */ - protected abstract String getQueryLastIndexDate(); - - /** - * @return name of the query which will return all current ids. - */ - protected abstract String getQueryAllIds(); -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java deleted file mode 100644 index 01599a4..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexDocumentsTask.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks; - -import com.redhat.satellite.search.config.Configuration; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.scheduler.tasks.crawl.WebCrawl; - -import org.apache.log4j.Logger; - -import org.quartz.JobDataMap; -import org.quartz.StatefulJob; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -import java.io.File; -import java.io.IOException; - - -/** - * Task to index help documents - * - * @version $Rev: $ - */ -public class IndexDocumentsTask implements StatefulJob { - // We do _not_ want this task to run concurrently with itself, - // therefore using StatefulJob. - private static Logger log = Logger.getLogger(IndexDocumentsTask.class); - public static final String TASK_REINDEX = new String("TASK.RE-INDEX"); - public static final String TASK_COMPLETE = new String("TASK.COMPLETE"); - - - /** - * {@inheritDoc} - */ - public void execute(JobExecutionContext ctx) - throws JobExecutionException { - JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); - IndexManager indexManager = (IndexManager) jobData.get("indexManager"); - - String indexWorkDir = indexManager.getIndexWorkDir(); - if ((indexWorkDir == null) || (indexWorkDir.compareTo("") == 0)) { - throw new JobExecutionException("indexWorkDir invalid"); - } - File docsIndexDir = new File(indexWorkDir + File.separator + - IndexManager.DOCS_INDEX_NAME); - File indexCheck = new File(docsIndexDir, TASK_REINDEX); - File success = new File(docsIndexDir, TASK_COMPLETE); - - boolean reindex = false; - - log.info("task running"); - - if (!docsIndexDir.exists()) { - // first time running, the index needs to be created from scratch - log.info("Creating directory: " + docsIndexDir.getPath()); - if (!docsIndexDir.mkdirs()) { - throw new JobExecutionException("Unable to create dir: " + - docsIndexDir.getPath()); - } - reindex = true; - } - else { - // Will reindex if a TASK.RE-INDEX is present or - // we don't see TASK.COMPLETE - log.info("docsIndexDir<" + docsIndexDir + "> exists"); - if (indexCheck.exists()) { - log.info("Found (" + indexCheck.getPath() + - "), Index needs to be updated."); - reindex = true; - } - else if (!success.exists()) { - log.info("Index exists, but it appears to be incomplete, missing " + - success.getPath()); - reindex = true; - } - } - - if (reindex) { - //We need to communicate index is incomplete by removing the success file - - if (success.exists()) { - if (!success.delete()) { - throw new JobExecutionException("Unable to delete file: " + - success.getPath()); - } - } - if ((indexCheck != null) && (indexCheck.exists())) { - // cleanup the file which triggered the re-index - if (!indexCheck.delete()) { - throw new JobExecutionException("Unable to delete file: " + - indexCheck.getPath()); - } - } - try { - Configuration config = (Configuration)jobData.get("configuration"); - if (!reIndexDocs(docsIndexDir.getPath(), config)) { - log.error("Failed reindexing documents to : "); - throw new JobExecutionException("Reindexing of Documents failed."); - } - //Mark that docs have been indexed and are ready to be used. - success.createNewFile(); - } - catch (IOException e) { - throw new JobExecutionException(e); - } - } - } - - protected boolean reIndexDocs(String docsIndexDir, Configuration config) - throws IOException { - - int threads = config.getInt("search.nutch.threads", 10); - int depth = config.getInt("search.nutch.depth", 5); - String inputUrlFile = config.getString("search.nutch.inputUrlFile", "nutch/urls"); - String tmpCrawlDir = config.getString("search.nutch.tmpDir", "/tmp/crawl_output"); - - WebCrawl wCrawl = new WebCrawl(); - wCrawl.setDepth(depth); - wCrawl.setInputUrlFile(inputUrlFile); - wCrawl.setOutputIndexDir(docsIndexDir); - wCrawl.setThreads(threads); - wCrawl.setTmpCrawlDir(tmpCrawlDir); - - return wCrawl.crawl(); - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java deleted file mode 100644 index 1c3ef6b..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexErrataTask.java +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.Query; -import com.redhat.satellite.search.db.WriteQuery; -import com.redhat.satellite.search.db.models.Errata; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; - -import org.apache.log4j.Logger; -import org.apache.lucene.document.Document; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -import java.sql.SQLException; -import java.util.Calendar; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - - -/** - * IndexErrataTask - * @version $Rev$ - */ -public class IndexErrataTask implements Job { - - private static Logger log = Logger.getLogger(IndexErrataTask.class); - private String lang = "en"; - /** - * {@inheritDoc} - */ - public void execute(JobExecutionContext ctx) - throws JobExecutionException { - JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); - DatabaseManager databaseManager = - (DatabaseManager)jobData.get("databaseManager"); - IndexManager indexManager = - (IndexManager)jobData.get("indexManager"); - - try { - - List<Errata> errata = getErrata(databaseManager); - int count = 0; - log.info("found [" + errata.size() + "] errata to index"); - for (Iterator<Errata> iter = errata.iterator(); iter.hasNext();) { - Errata current = iter.next(); - indexErrata(indexManager, current); - count++; - if (count == 10 || !iter.hasNext()) { - if (System.getProperties().get("isTesting") == null) { - updateLastErrataId(databaseManager, current.getId()); - } - count = 0; - } - } - } - catch (SQLException e) { - throw new JobExecutionException(e); - } - catch (IndexingException e) { - throw new JobExecutionException(e); - } - } - /** - * @param databaseManager - * @param sid - */ - private void updateLastErrataId(DatabaseManager databaseManager, long eid) - throws SQLException { - - WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastErrata"); - WriteQuery insertQuery = databaseManager.getWriterQuery("createLastErrata"); - - try { - Map params = new HashMap(); - params.put("id", eid); - params.put("last_modified", Calendar.getInstance().getTime()); - if (updateQuery.update(params) == 0) { - insertQuery.insert(params); - } - } - finally { - try { - if (updateQuery != null) { - updateQuery.close(); - } - } - finally { - if (insertQuery != null) { - insertQuery.close(); - } - } - } - } - - /** - * @param indexManager - * @param current - */ - private void indexErrata(IndexManager indexManager, Errata errata) - throws IndexingException { - - Map<String, String> attrs = new HashMap<String, String>(); - attrs.put("id", new Long(errata.getId()).toString()); - attrs.put("advisory", errata.getAdvisory()); - attrs.put("advisoryType", errata.getAdvisoryType()); - attrs.put("advisoryName", errata.getAdvisoryName()); - attrs.put("advisoryRel", new Long(errata.getAdvisoryRel()).toString()); - attrs.put("product", errata.getProduct()); - attrs.put("description", errata.getDescription()); - attrs.put("synopsis", errata.getSynopsis()); - attrs.put("topic", errata.getTopic()); - attrs.put("solution", errata.getSolution()); - attrs.put("issueDate", errata.getIssueDate()); - attrs.put("updateDate", errata.getUpdateDate()); - attrs.put("notes", errata.getNotes()); - attrs.put("orgId", errata.getOrgId()); - attrs.put("created", errata.getCreated()); - attrs.put("modified", errata.getModified()); - attrs.put("lastModified", errata.getLastModified()); - attrs.put("name", errata.getAdvisory()); - - log.info("Indexing errata: " + errata.getId() + ": " + attrs.toString()); - DocumentBuilder edb = BuilderFactory.getBuilder(BuilderFactory.ERRATA_TYPE); - Document doc = edb.buildDocument(new Long(errata.getId()), attrs); - indexManager.addToIndex("errata", doc, lang); - } - - /** - * @param databaseManager - * @return - */ - private List<Errata> getErrata(DatabaseManager databaseManager) - throws SQLException { - - List<Errata> retval = null; - Query<Long> query = databaseManager.getQuery("getLastErrataId"); - Long eid = null; - try { - eid = query.load(); - } - finally { - query.close(); - } - if (eid == null) { - eid = new Long(0); - } - Query<Errata> errataQuery = databaseManager.getQuery("listErrataFromId"); - try { - retval = errataQuery.loadList(eid); - } - finally { - errataQuery.close(); - } - return retval; - } - -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java deleted file mode 100644 index 8960d94..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexHardwareDevicesTask.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks; - -import com.redhat.satellite.search.db.models.GenericRecord; -import com.redhat.satellite.search.db.models.HardwareDevice; -import com.redhat.satellite.search.index.builder.BuilderFactory; - -import org.apache.log4j.Logger; - -import java.util.HashMap; -import java.util.Map; - -/** - * IndexHardwareDevicesTask - * @version $Rev$ - */ -public class IndexHardwareDevicesTask extends GenericIndexTask { - - private static Logger log = Logger.getLogger(IndexHardwareDevicesTask.class); - - /** - * {@inheritDoc} - */ - @Override - protected Map<String, String> getFieldMap(GenericRecord data) - throws ClassCastException { - HardwareDevice dev = (HardwareDevice)data; - Map<String, String> attrs = new HashMap<String, String>(); - attrs.put("serverId", new Long(dev.getServerId()).toString()); - attrs.put("classInfo", dev.getClassInfo()); - attrs.put("bus", dev.getBus()); - attrs.put("detached", new Long(dev.getDetached()).toString()); - attrs.put("device", dev.getDevice()); - attrs.put("driver", dev.getDriver()); - attrs.put("description", dev.getDescription()); - attrs.put("vendorId", dev.getVendorId()); - attrs.put("deviceId", dev.getDeviceId()); - attrs.put("subVendorId", dev.getSubVendorId()); - attrs.put("subDeviceId", dev.getSubDeviceId()); - attrs.put("pciType", new Long(dev.getPciType()).toString()); - return attrs; - } - - /** - * {@inheritDoc} - */ - @Override - public String getIndexName() { - return BuilderFactory.HARDWARE_DEVICE_TYPE; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryCreateLastRecord() { - return "createLastHardwareDevice"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastRecord() { - return "getLastHardwareDeviceId"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastIndexDate() { - return "getLastHardwareDeviceIndexRun"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryRecordsToIndex() { - return "getHardwareDeviceById"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryUpdateLastRecord() { - return "updateLastHardwareDevice"; - } - - /** - * {@inheritDoc} - */ - public String getUniqueFieldId() { - return "id"; - } - /** - * {@inheritDoc} - */ - public String getQueryAllIds() { - return "queryAllHwDeviceIds"; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java deleted file mode 100644 index 7c657e6..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexPackagesTask.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ - -package com.redhat.satellite.search.scheduler.tasks; - -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.Query; -import com.redhat.satellite.search.db.WriteQuery; -import com.redhat.satellite.search.db.models.RhnPackage; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.index.builder.DocumentBuilder; - -import org.apache.log4j.Logger; -import org.apache.lucene.document.Document; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -import java.sql.SQLException; -import java.util.Calendar; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * Task to index package information - * - * @version $Rev$ - */ -public class IndexPackagesTask implements Job { - - private static Logger log = Logger.getLogger(IndexPackagesTask.class); - private String lang = "en"; - /** - * {@inheritDoc} - */ - public void execute(JobExecutionContext ctx) throws JobExecutionException { - JobDataMap jobData = ctx.getJobDetail().getJobDataMap(); - DatabaseManager databaseManager = - (DatabaseManager)jobData.get("databaseManager"); - IndexManager indexManager = - (IndexManager)jobData.get("indexManager"); - - try { - if (System.getProperties().get("isTesting") != null) { - cleanLastPackage(databaseManager); - } - List<RhnPackage> packages = getPackages(databaseManager); - int count = 0; - log.info("found [" + packages.size() + "] packages to index"); - for (Iterator<RhnPackage> iter = packages.iterator(); iter.hasNext();) { - RhnPackage current = iter.next(); - indexPackage(indexManager, current); - count++; - if (count == 10 || !iter.hasNext()) { - if (System.getProperties().get("isTesting") == null) { - updateLastPackageId(databaseManager, current.getId()); - } - count = 0; - } - } - } - catch (SQLException e) { - throw new JobExecutionException(e); - } - catch (IndexingException e) { - throw new JobExecutionException(e); - } - } - - private void cleanLastPackage(DatabaseManager databaseManager) throws SQLException { - WriteQuery query = null; - try { - query = databaseManager.getWriterQuery("deleteLastPackage"); - query.delete(null); - } - finally { - query.close(); - } - } - - private void indexPackage(IndexManager indexManager, RhnPackage pkg) - throws IndexingException { - Map<String, String> attrs = new HashMap<String, String>(); - attrs.put("name", pkg.getName()); - attrs.put("version", pkg.getVersion()); - attrs.put("release", pkg.getRelease()); - attrs.put("filename", pkg.getFileName()); - attrs.put("description", pkg.getDescription()); - attrs.put("summary", pkg.getSummary()); - attrs.put("arch", pkg.getArch()); - log.info("Indexing package: " + pkg.getId() + ": " + attrs.toString()); - DocumentBuilder pdb = BuilderFactory.getBuilder(BuilderFactory.PACKAGES_TYPE); - Document doc = pdb.buildDocument(new Long(pkg.getId()), attrs); - indexManager.addToIndex("package", doc, lang); - } - - private void updateLastPackageId(DatabaseManager databaseManager, - Long packageId) throws SQLException { - WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastPackage"); - WriteQuery insertQuery = databaseManager.getWriterQuery("createLastPackage"); - try { - Map params = new HashMap(); - params.put("id", packageId); - params.put("last_modified", Calendar.getInstance().getTime()); - if (updateQuery.update(params) == 0) { - insertQuery.insert(params); - } - } - finally { - try { - if (updateQuery != null) { - updateQuery.close(); - } - } - finally { - if (insertQuery != null) { - insertQuery.close(); - } - } - } - } - - private List<RhnPackage> getPackages(DatabaseManager databaseManager) - throws SQLException { - List<RhnPackage> retval = null; - Query<Long> query = databaseManager.getQuery("getLastPackageId"); - Long packageId = null; - try { - packageId = query.load(); - } - finally { - query.close(); - } - if (packageId == null) { - packageId = new Long(0); - } - Query<RhnPackage> pkgQuery = databaseManager.getQuery("listPackagesFromId"); - try { - retval = pkgQuery.loadList(packageId); - } - finally { - pkgQuery.close(); - } - return retval; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java deleted file mode 100644 index ef7b366..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexServerCustomInfoTask.java +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks; - -import com.ibatis.sqlmap.client.SqlMapException; -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.Query; -import com.redhat.satellite.search.db.models.GenericRecord; -import com.redhat.satellite.search.db.models.ServerCustomInfo; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.builder.BuilderFactory; - -import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -import org.apache.log4j.Logger; - -/** - * IndexServerCustomInfoTask - * @version $Rev$ - * - */ -public class IndexServerCustomInfoTask extends GenericIndexTask { - private static Logger log = Logger.getLogger(IndexServerCustomInfoTask.class); - - /** - * Will determine if any records have been deleted from the DB, then will - * delete those records from the lucene index. - * @return number of deleted records - */ - protected int handleDeletedRecords(DatabaseManager databaseManager, - IndexManager indexManager) - throws SQLException { - List<ServerCustomInfo> ids = null; - Query<ServerCustomInfo> query = null; - String uniqField = null; - String indexName = null; - HashSet<String> idSet = null; - try { - query = databaseManager.getQuery(getQueryAllIds()); - ids = query.loadList(Collections.EMPTY_MAP); - if ((ids == null) || (ids.size() == 0)) { - log.info("Got back no data from '" + getQueryAllIds() + "'"); - log.info("Skipping the handleDeletedRecords() method"); - return 0; - } - idSet = new HashSet(); - for (ServerCustomInfo scInfo : ids) { - idSet.add(scInfo.getUniqId()); - } - uniqField = getUniqueFieldId(); - indexName = getIndexName(); - } - catch (SqlMapException e) { - e.printStackTrace(); - log.info("Error with 'getQueryAllIds()' on " + - super.getClass().toString()); - //just print the warning so we know and skip this method. - return 0; - } - finally { - if (query != null) { - query.close(); - } - } - return indexManager.deleteRecordsNotInList(idSet, indexName, uniqField); - } - - - - - /** - * {@inheritDoc} - */ - @Override - protected Map<String, String> getFieldMap(GenericRecord data) - throws ClassCastException { - ServerCustomInfo scInfo = (ServerCustomInfo)data; - Map<String, String> attrs = new HashMap<String, String>(); - attrs.put("uniqId", scInfo.getUniqId()); - attrs.put("serverId", new Long(scInfo.getServerId()).toString()); - attrs.put("value", scInfo.getValue()); - attrs.put("label", scInfo.getLabel()); - attrs.put("createdBy", new Long(scInfo.getCreatedBy()).toString()); - attrs.put("lastModifiedBy", new Long(scInfo.getLastModifiedBy()).toString()); - attrs.put("created", scInfo.getCreated()); - attrs.put("modified", scInfo.getModified()); - return attrs; - } - - /** - * {@inheritDoc} - */ - @Override - public String getIndexName() { - return BuilderFactory.SERVER_CUSTOM_INFO_TYPE; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryCreateLastRecord() { - return "createLastServerCustomInfo"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastRecord() { - return "getLastServerCustomInfoId"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastIndexDate() { - return "getLastServerCustomInfoIndexRun"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryRecordsToIndex() { - return "getServerCustomInfoByIdOrDate"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryUpdateLastRecord() { - return "updateLastServerCustomInfo"; - } - - /** - * {@inheritDoc} - */ - public String getUniqueFieldId() { - return "uniqId"; - } - /** - * {@inheritDoc} - */ - public String getQueryAllIds() { - return "queryAllServerCustomInfoIds"; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java deleted file mode 100644 index 611f1c6..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSnapshotTagsTask.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks; - -import com.redhat.satellite.search.db.models.GenericRecord; -import com.redhat.satellite.search.db.models.SnapshotTag; -import com.redhat.satellite.search.index.builder.BuilderFactory; - -import java.util.HashMap; -import java.util.Map; - -/** - * IndexSnapshotTagsTask - * @version $Rev$ - * - */ -public class IndexSnapshotTagsTask extends GenericIndexTask { - - /** - * {@inheritDoc} - */ - @Override - protected Map<String, String> getFieldMap(GenericRecord data) - throws ClassCastException { - SnapshotTag sTag = (SnapshotTag)data; - Map<String, String> attrs = new HashMap<String, String>(); - attrs.put("snapshotId", new Long(sTag.getSnapshotId()).toString()); - attrs.put("tagNameId", new Long(sTag.getTagNameId()).toString()); - attrs.put("serverId", new Long(sTag.getServerId()).toString()); - attrs.put("orgId", new Long(sTag.getOrgId()).toString()); - attrs.put("name", sTag.getName()); - attrs.put("created", sTag.getCreated()); - attrs.put("modified", sTag.getModified()); - return attrs; - } - - /** - * {@inheritDoc} - */ - @Override - public String getIndexName() { - return BuilderFactory.SNAPSHOT_TAG_TYPE; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryCreateLastRecord() { - return "createLastSnapshotTag"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastRecord() { - return "getLastSnapshotTagId"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastIndexDate() { - return "getLastSnapshotTagIndexRun"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryRecordsToIndex() { - return "getSnapshotTagByIdOrDate"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryUpdateLastRecord() { - return "updateLastSnapshotTag"; - } - - /** - * {@inheritDoc} - */ - public String getUniqueFieldId() { - return "tagNameId"; - } - - /** - * {@inheritDoc} - */ - public String getQueryAllIds() { - return "queryAllSnapshotTagIds"; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java deleted file mode 100644 index 4cb071b..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/IndexSystemsTask.java +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks; - - -import com.redhat.satellite.search.db.models.GenericRecord; -import com.redhat.satellite.search.db.models.Server; -import com.redhat.satellite.search.index.builder.BuilderFactory; - -import java.util.HashMap; - -import java.util.Map; - - -// Main tasks: -// 1) Index new systems, i.e. system id is greater than last recorded system id indexed. -// 2) Update the existing index of systems which have been modified -// 3) Remove systems which have been deleted from the system. - -/** - * IndexSystemsTask - * @version $Rev$ - */ -public class IndexSystemsTask extends GenericIndexTask { - /** - * {@inheritDoc} - */ - @Override - protected Map<String, String> getFieldMap(GenericRecord data) - throws ClassCastException { - Server srvr = (Server)data; - - Map<String, String> attrs = new HashMap<String, String>(); - attrs.put("id", new Long(srvr.getId()).toString()); - attrs.put("system_id", new Long(srvr.getId()).toString()); - attrs.put("name", srvr.getName()); - attrs.put("description", srvr.getDescription()); - attrs.put("info", srvr.getInfo()); - attrs.put("runningKernel", srvr.getRunningKernel()); - attrs.put("machine", srvr.getMachine()); - attrs.put("rack", srvr.getRack()); - attrs.put("room", srvr.getRoom()); - attrs.put("building", srvr.getBuilding()); - attrs.put("address1", srvr.getAddress1()); - attrs.put("address2", srvr.getAddress2()); - attrs.put("city", srvr.getCity()); - attrs.put("state", srvr.getState()); - attrs.put("country", srvr.getCountry()); - attrs.put("hostname", srvr.getHostname()); - attrs.put("ipaddr", srvr.getIpaddr()); - attrs.put("dmiVendor", srvr.getDmiVendor()); - attrs.put("dmiSystem", srvr.getDmiSystem()); - attrs.put("dmiProduct", srvr.getDmiProduct()); - attrs.put("dmiBiosVendor", srvr.getDmiBiosVendor()); - attrs.put("dmiBiosVersion", srvr.getDmiBiosVersion()); - attrs.put("dmiBiosRelease", srvr.getDmiBiosRelease()); - attrs.put("dmiAsset", srvr.getDmiAsset()); - attrs.put("dmiBoard", srvr.getDmiBoard()); - attrs.put("cpuBogoMIPs", srvr.getCpuBogoMIPS()); - attrs.put("cpuCache", srvr.getCpuCache()); - attrs.put("cpuFamily", srvr.getCpuFamily()); - attrs.put("cpuMhz", srvr.getCpuMhz()); - attrs.put("cpuStepping", srvr.getCpuStepping()); - attrs.put("cpuFlags", srvr.getCpuFlags()); - attrs.put("cpuModel", srvr.getCpuModel()); - attrs.put("cpuVersion", srvr.getCpuVersion()); - attrs.put("cpuVendor", srvr.getCpuVendor()); - if (srvr.getCpuNumberOfCpus() != null) { - attrs.put("cpuNumberOfCpus", srvr.getCpuNumberOfCpus().toString()); - } - attrs.put("cpuAcpiVersion", srvr.getCpuAcpiVersion()); - attrs.put("cpuApic", srvr.getCpuApic()); - attrs.put("cpuApmVersion", srvr.getCpuApmVersion()); - attrs.put("cpuChipset", srvr.getCpuChipset()); - attrs.put("checkin", srvr.getCheckin()); - attrs.put("registered", srvr.getRegistered()); - attrs.put("ram", srvr.getRam()); - attrs.put("swap", srvr.getSwap()); - return attrs; - } - - /** - * {@inheritDoc} - */ - @Override - public String getIndexName() { - return BuilderFactory.SERVER_TYPE; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryCreateLastRecord() { - return "createLastServer"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastRecord() { - return "getLastServerId"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryLastIndexDate() { - return "getLastServerIndexRun"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryRecordsToIndex() { - return "getServerByIdOrDate"; - } - - /** - * {@inheritDoc} - */ - @Override - protected String getQueryUpdateLastRecord() { - return "updateLastServer"; - } - - /** - * {@inheritDoc} - */ - public String getUniqueFieldId() { - return "id"; - } - - /** - * {@inheritDoc} - */ - public String getQueryAllIds() { - return "queryAllServerIds"; - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java deleted file mode 100644 index 9352f3c..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexErrataTaskTest.java +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks.tests; - -import com.redhat.satellite.search.config.Configuration; -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.WriteQuery; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.scheduler.tasks.IndexErrataTask; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; - -import org.apache.lucene.document.Document; -import org.quartz.JobDataMap; -import org.quartz.JobDetail; -import org.quartz.JobExecutionContext; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.SimpleTrigger; -import org.quartz.Trigger; -import org.quartz.TriggerListener; -import org.quartz.impl.StdSchedulerFactory; - -import org.apache.log4j.Logger; - -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - - -/** - * IndexErrataTaskTest - * @version $Rev$ - */ -public class IndexErrataTaskTest extends BaseTestCase { - private static Logger log = Logger.getLogger(IndexErrataTaskTest.class); - private Scheduler scheduler; - - public void tearDown() throws Exception { - DatabaseManager databaseManager = (DatabaseManager) - container.getComponentInstanceOfType(DatabaseManager.class); - WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastErrata"); - Map params = new HashMap(); - params.put("id", 0L); - params.put("last_modified", new Date(0)); - updateQuery.update(params); - updateQuery.close(); - if (scheduler != null) { - scheduler.shutdown(false); - } - super.tearDown(); - } - - public void testExecute() throws InterruptedException, SchedulerException { - - DatabaseManager databaseManager = (DatabaseManager) - container.getComponentInstanceOfType(DatabaseManager.class); - - IndexManager indexManager = (IndexManager) - container.getComponentInstanceOfType(IndexManager.class); - - try { - scheduler = StdSchedulerFactory.getDefaultScheduler(); - } - catch (SchedulerException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - Trigger trigger = new SimpleTrigger("index", - "default", - "index", - "default", - new Date(), - null, - 0, - 100); - trigger.setMisfireInstruction( - SimpleTrigger.MISFIRE_INSTRUCTION_FIRE_NOW); - JobDetail detail = new JobDetail("index", "default", IndexErrataTask.class); - JobDataMap jobData = new JobDataMap(); - jobData.put("indexManager", indexManager); - jobData.put("databaseManager", databaseManager); - detail.setJobDataMap(jobData); - try { - scheduler.addTriggerListener(new TestTrigger()); - scheduler.scheduleJob(detail, trigger); - } - catch (SchedulerException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - try { - - scheduler.start(); - } - catch (SchedulerException e) { - System.out.println("woo hoo"); - } - Thread.sleep(1000); - if(!scheduler.isShutdown()) { - scheduler.shutdown(true); - } - System.out.println("running"); - } - - /** - * {@inheritDoc} - */ - @Override - protected Class[] getComponentClasses() { - Class[] comps = {DatabaseManager.class, TestIndexManager.class}; - return TestUtil.buildComponentsList(comps); - } - - public static class TestIndexManager extends IndexManager { - - @Override - public void addToIndex(String indexName, Document doc, String lang) throws IndexingException { - assertNotNull(doc); - assertNotNull(doc.getField("advisory")); - assertNotNull(doc.getField("advisory").stringValue()); - log.info("idx[" + indexName + "] doc [" + doc.toString() + "]"); - } - - /** - * @param config - */ - public TestIndexManager(Configuration config) { - super(config); - } - - } - - public static class TestTrigger implements TriggerListener { - - /** - * {@inheritDoc} - */ - public String getName() { - return "test trigger"; - } - - /** - * {@inheritDoc} - */ - public void triggerComplete(Trigger trigger, JobExecutionContext context, int triggerInstructionCode) { - System.out.println("**************************** completed"); - } - - /** - * {@inheritDoc} - */ - public void triggerFired(Trigger trigger, JobExecutionContext context) { - System.out.println("**************************** fired"); - } - - /** - * {@inheritDoc} - */ - public void triggerMisfired(Trigger trigger) { - System.out.println("**************************** misfire"); - } - - /** - * {@inheritDoc} - */ - public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) { - return false; - } - - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java deleted file mode 100644 index 2f43854..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tasks/tests/IndexSystemsTaskTest.java +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Copyright (c) 2008 Red Hat, Inc. - * - * This software is licensed to you under the GNU General Public License, - * version 2 (GPLv2). There is NO WARRANTY for this software, express or - * implied, including the implied warranties of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 - * along with this software; if not, see - * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - * - * Red Hat trademarks are not licensed under GPLv2. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package com.redhat.satellite.search.scheduler.tasks.tests; - -import com.redhat.satellite.search.config.Configuration; -import com.redhat.satellite.search.db.DatabaseManager; -import com.redhat.satellite.search.db.WriteQuery; -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.scheduler.tasks.IndexSystemsTask; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; - -import org.apache.lucene.document.Document; -import org.quartz.JobDataMap; -import org.quartz.JobDetail; -import org.quartz.JobExecutionContext; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.SimpleTrigger; -import org.quartz.Trigger; -import org.quartz.TriggerListener; -import org.quartz.impl.StdSchedulerFactory; - -import org.apache.log4j.Logger; - -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - - -/** - * IndexSystemsTaskTest - * @version $Rev$ - */ -public class IndexSystemsTaskTest extends BaseTestCase { - private static Logger log = Logger.getLogger(IndexSystemsTaskTest.class); - private Scheduler scheduler; - - public void tearDown() throws Exception { - DatabaseManager databaseManager = (DatabaseManager) - container.getComponentInstanceOfType(DatabaseManager.class); - WriteQuery updateQuery = databaseManager.getWriterQuery("updateLastServer"); - Map params = new HashMap(); - params.put("id", 0L); - params.put("last_modified", new Date(0)); - updateQuery.update(params); - updateQuery.close(); - scheduler.shutdown(false); - - super.tearDown(); - } - - public void testExecute() throws InterruptedException, SchedulerException { - - DatabaseManager databaseManager = (DatabaseManager) - container.getComponentInstanceOfType(DatabaseManager.class); - - IndexManager indexManager = (IndexManager) - container.getComponentInstanceOfType(IndexManager.class); - - try { - scheduler = StdSchedulerFactory.getDefaultScheduler(); - } - catch (SchedulerException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - Trigger trigger = new SimpleTrigger("index", - "default", - "index", - "default", - new Date(), - null, - 0, - 100); - trigger.setMisfireInstruction( - SimpleTrigger.MISFIRE_INSTRUCTION_FIRE_NOW); - JobDetail detail = new JobDetail("index", "default", IndexSystemsTask.class); - JobDataMap jobData = new JobDataMap(); - jobData.put("indexManager", indexManager); - jobData.put("databaseManager", databaseManager); - detail.setJobDataMap(jobData); - try { - scheduler.addTriggerListener(new TestTrigger()); - scheduler.scheduleJob(detail, trigger); - } - catch (SchedulerException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - try { - - scheduler.start(); - } - catch (SchedulerException e) { - System.out.println("woo hoo"); - } - Thread.sleep(1000); - if(!scheduler.isShutdown()) { - scheduler.shutdown(true); - } - System.out.println("running"); - } - - /** - * {@inheritDoc} - */ - @Override - protected Class[] getComponentClasses() { - Class[] comps = {DatabaseManager.class, TestIndexManager.class}; - return TestUtil.buildComponentsList(comps); - } - - public static class TestIndexManager extends IndexManager { - - @Override - public void addToIndex(String indexName, Document doc, String lang) throws IndexingException { - assertNotNull(doc); - assertNotNull(doc.getField("id")); - assertNotNull(doc.getField("name").stringValue()); - assertNotNull(doc.getField("description").stringValue()); - assertNotNull(doc.getField("info").stringValue()); - log.info("idx[" + indexName + "] doc [" + doc.toString() + "]"); - } - - /** - * @param config - */ - public TestIndexManager(Configuration config) { - super(config); - } - - } - - public static class TestTrigger implements TriggerListener { - - /** - * {@inheritDoc} - */ - public String getName() { - return "test trigger"; - } - - /** - * {@inheritDoc} - */ - public void triggerComplete(Trigger trigger, JobExecutionContext context, int triggerInstructionCode) { - System.out.println("**************************** completed"); - } - - /** - * {@inheritDoc} - */ - public void triggerFired(Trigger trigger, JobExecutionContext context) { - System.out.println("**************************** fired"); - } - - /** - * {@inheritDoc} - */ - public void triggerMisfired(Trigger trigger) { - System.out.println("**************************** misfire"); - } - - /** - * {@inheritDoc} - */ - public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) { - return false; - } - - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java b/search-server/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java deleted file mode 100644 index 382c3c7..0000000 --- a/search-server/src/java/com/redhat/satellite/search/scheduler/tests/ScheduleManagerTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.redhat.satellite.search.scheduler.tests; - -import com.redhat.satellite.search.index.IndexManager; -import com.redhat.satellite.search.index.IndexingException; -import com.redhat.satellite.search.index.QueryParseException; -import com.redhat.satellite.search.index.Result; -import com.redhat.satellite.search.index.builder.BuilderFactory; -import com.redhat.satellite.search.scheduler.ScheduleManager; -import com.redhat.satellite.search.tests.BaseTestCase; -import com.redhat.satellite.search.tests.TestUtil; - -import java.util.List; - -public class ScheduleManagerTest extends BaseTestCase { - @SuppressWarnings("unchecked") - @Override - protected Class[] getComponentClasses() { - return TestUtil.buildComponentsList(ScheduleManager.class); - } - - public void testIndexing() throws IndexingException, QueryParseException { - try { - Thread.sleep(30000); - IndexManager mgr = (IndexManager) - container.getComponentInstance(IndexManager.class); - List<Result>hits = mgr.search("package", "description:package", - "en"); - assertTrue(hits.size() > 0); - } - catch (InterruptedException e) { - return; - } - } - - public void testTriggerIndexTask() { - ScheduleManager sm = new ScheduleManager(null, null); - assertTrue(sm.triggerIndexTask(BuilderFactory.ERRATA_TYPE)); - assertTrue(sm.triggerIndexTask(BuilderFactory.HARDWARE_DEVICE_TYPE)); - assertTrue(sm.triggerIndexTask(BuilderFactory.PACKAGES_TYPE)); - assertTrue(sm.triggerIndexTask(BuilderFactory.SERVER_CUSTOM_INFO_TYPE)); - assertTrue(sm.triggerIndexTask(BuilderFactory.SERVER_TYPE)); - assertTrue(sm.triggerIndexTask(BuilderFactory.SNAPSHOT_TAG_TYPE)); - assertFalse(sm.triggerIndexTask(BuilderFactory.DOCS_TYPE)); - assertFalse(sm.triggerIndexTask(null)); - assertFalse(sm.triggerIndexTask("biteme")); - assertFalse(sm.triggerIndexTask("")); - } -} diff --git a/search-server/src/java/com/redhat/satellite/search/tests/BaseTestCase.java b/search-server/src/java/com/redhat/satellite/search/tests/BaseTestCase.java deleted file mode 100644 index 15e6e90..0000000 --- a/search-server/src/java/com/redhat/satellite/search/tests/BaseTestCase.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.redhat.satellite.search.tests; - -import com.redhat.satellite.search.config.Configuration; - -import org.picocontainer.defaults.DefaultPicoContainer; - -import junit.framework.TestCase; - -public abstract class BaseTestCase extends TestCase { - - protected DefaultPicoContainer container; - - @Override - protected void setUp() throws Exception { - System.getProperties().put("isTesting", "true"); - super.setUp(); - container = TestUtil.buildContainer(getComponentClasses()); - container.start(); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - Configuration config = (Configuration) - container.getComponentInstanceOfType(Configuration.class); - TestUtil.cleanupDirectories(config); - container.stop(); - } - - @SuppressWarnings("unchecked") - protected abstract Class[] getComponentClasses(); - -} diff --git a/search-server/src/java/com/redhat/satellite/search/tests/TestUtil.java b/search-server/src/java/com/redhat/satellite/search/tests/TestUtil.java deleted file mode 100644 index b86d80f..0000000 --- a/search-server/src/java/com/redhat/satellite/search/tests/TestUtil.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.redhat.satellite.search.tests; - -import com.redhat.satellite.search.config.Configuration; - -import org.picocontainer.defaults.DefaultPicoContainer; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.StringReader; -import java.lang.reflect.Constructor; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -public class TestUtil { - - private TestUtil() { - - } - - @SuppressWarnings("unchecked") - public static DefaultPicoContainer buildContainer(Class[] components) throws IOException { - Configuration config = makeConfiguration(); - DefaultPicoContainer container = new DefaultPicoContainer(); - container.registerComponentInstance("appConfig", config); - for (int x = 0; x < components.length; x++) { - container.registerComponentImplementation(components[x]); - } - return container; - } - - @SuppressWarnings("unchecked") - public static Class[] buildComponentsList(Class component) { - Class[] arg = {component}; - return buildComponentsList(arg); - } - - @SuppressWarnings("unchecked") - public static Class[] buildComponentsList(Class[] components) { - List<Class>accum = new ArrayList<Class>(); - for (int x = 0; x < components.length; x++) { - processComponent(components[x], accum); - } - Class[] retval = new Class[accum.size()]; - accum.toArray(retval); - return retval; - } - - public static Configuration makeConfiguration() throws IOException { - String prefix = "search-server-test"; - String indexWorkDir = generateTempDir(prefix); - StringBuilder builder = new StringBuilder(); - builder.append("search.index_work_dir=").append(indexWorkDir).append("\n"); - builder.append("search.rpc_port=").append(generatePortNumber()).append("\n"); - builder.append("search.rpc_handlers="); - builder.append("index:com.redhat.satellite.search.rpc.handlers.IndexHandler\n\n"); - - // see if we overrode any properties in test.properties - File f = new File("src/config/test.properties"); - if (f.exists()) { - FileReader fr = new FileReader(f); - BufferedReader br = new BufferedReader(fr); - String line = null; - while ((line = br.readLine()) != null) { - builder.append(line); - builder.append("\n"); - } - } - - StringReader sr = new StringReader(builder.toString()); - return new Configuration(new BufferedReader(sr)); - } - - public static void cleanupDirectories(Configuration config) - throws IOException { - String workDir = config.getString("search.index_work_dir", null); - if (workDir != null) { - File f = new File(workDir); - if (f.exists() && f.isDirectory()) { - Runtime.getRuntime().exec("rm -rf " + workDir); - } - } - } - @SuppressWarnings("unchecked") - private static List<Class> processComponent(Class component, List<Class>accum) { - if (component == Configuration.class) { - return accum; - } - if (accum.indexOf(component) == -1) { - accum.add(component); - } - Constructor[] constructors = component.getConstructors(); - if (constructors != null) { - for (int x = 0;x < constructors.length; x++) { - Class[] paramTypes = constructors[x].getParameterTypes(); - if (paramTypes.length > 0) { - for (int y = 0;y < paramTypes.length; y++) { - if (isCandidate(paramTypes[y].getName())) { - processComponent(paramTypes[y], accum); - } - } - } - } - } - return accum; - } - - private static boolean isCandidate(String className) { - boolean retval = true; - if (className.startsWith("java.") || className.startsWith("javax.") || - className.startsWith("com.sun.")) { - retval = false; - } - return retval; - } - - private static String generateTempDir(String prefix) throws IOException { - File f = File.createTempFile(prefix, null); - f.delete(); - return f.getAbsolutePath(); - } - - private static String generatePortNumber() { - Random r = new Random(System.nanoTime()); - int port = 0; - while (port < 1024) { - port = r.nextInt(20001); - } - return String.valueOf(port); - } -} diff --git a/search-server/task-lib/ant-contrib-1.0.jar b/search-server/task-lib/ant-contrib-1.0.jar deleted file mode 100644 index 50b884f..0000000 Binary files a/search-server/task-lib/ant-contrib-1.0.jar and /dev/null differ diff --git a/search-server/task-lib/antlr-2.7.6.jar b/search-server/task-lib/antlr-2.7.6.jar deleted file mode 100644 index 6f001d5..0000000 Binary files a/search-server/task-lib/antlr-2.7.6.jar and /dev/null differ diff --git a/search-server/task-lib/checkstyle-4.1.jar b/search-server/task-lib/checkstyle-4.1.jar deleted file mode 100644 index 49b639e..0000000 Binary files a/search-server/task-lib/checkstyle-4.1.jar and /dev/null differ diff --git a/search-server/task-lib/commons-beanutils-1.7.0.jar b/search-server/task-lib/commons-beanutils-1.7.0.jar deleted file mode 100644 index e024ab1..0000000 Binary files a/search-server/task-lib/commons-beanutils-1.7.0.jar and /dev/null differ diff --git a/search-server/task-lib/commons-logging-1.0.4.jar b/search-server/task-lib/commons-logging-1.0.4.jar deleted file mode 100644 index e2f8cee..0000000 Binary files a/search-server/task-lib/commons-logging-1.0.4.jar and /dev/null differ diff --git a/search-server/task-lib/emma-20040713.jar b/search-server/task-lib/emma-20040713.jar deleted file mode 100644 index 13e1118..0000000 Binary files a/search-server/task-lib/emma-20040713.jar and /dev/null differ diff --git a/search-server/task-lib/emma_ant-20040713.jar b/search-server/task-lib/emma_ant-20040713.jar deleted file mode 100644 index 76d47b8..0000000 Binary files a/search-server/task-lib/emma_ant-20040713.jar and /dev/null differ diff --git a/search-server/task-lib/ivy.jar b/search-server/task-lib/ivy.jar deleted file mode 100644 index 9421724..0000000 Binary files a/search-server/task-lib/ivy.jar and /dev/null differ diff --git a/search-server/task-lib/junit-3.8.1.jar b/search-server/task-lib/junit-3.8.1.jar deleted file mode 100644 index e2e4783..0000000 Binary files a/search-server/task-lib/junit-3.8.1.jar and /dev/null differ diff --git a/search-server/task-lib/regexp-1.3.jar b/search-server/task-lib/regexp-1.3.jar deleted file mode 100644 index 8c96d98..0000000 Binary files a/search-server/task-lib/regexp-1.3.jar and /dev/null differ