当前位置: 主页 > 浏览 >

Lucene2.9加入PaodingAnalyzer2.0,并实现stopwords

收藏 时间:2013-03-08 来源:查看 收藏:chriszt 阅读:506 标签:索引  文件  加入  搜索  指定  目录  
最近开始学习Lucene,java版本的lucene,入门还是比较快的,网上也很多关于lucene的学习资料。我在这里总结一下。 1. 搭建Lucene的开发环境:在classpath中添加lucene-core-2.9.1.jar包,在csdn可以下载到该jar包 2.加入庖丁解牛jar包 我用是庖丁解牛 2.0.0版本 下载地址:http://code.google.c

最近开始学习Lucene,java版本的lucene,入门还是比较快的,网上也很多关于lucene的学习资料。我在这里总结一下。

1. 搭建Lucene的开发环境:在classpath中添加lucene-core-2.9.1.jar包,在csdn可以下载到该jar包

2.加入庖丁解牛jar包

 

我用是庖丁解牛 2.0.0版本

下载地址:http://www.yeegle.com/links/gou/http_3A_2F_2Fcode.google.com_2Fp_2Fpaoding_2Fdownloads_2Flist.html
SVN地址:http://www.yeegle.com/links/gou/http_3A_2F_2Fpaoding.googlecode.com_2Fsvn_2Ftrunk_2Fpaoding-analysis_2F.html

里面有lucene-highlighter-2.2.0.jar, paoding-analysis.jar , commons-logging.jar , 相关的包要加入到classpath中

3。在lucene中加入庖丁解牛,配置文件

 

---------------------------------------------------------------

paoding-dic-home.properties

 

#values are "system-env" or "this";
#if value is "this" , using the paoding.dic.home as dicHome if configed!
#paoding.dic.home.config-fisrt=system-env


paoding.dic.home.config-fisrt=this

 

#dictionary home (directory)
#"classpath:xxx" means dictionary home is in classpath.
#e.g "classpath:dic" means dictionaries are in "classes/dic" directory or any other classpath directory

//下载庖丁包有该字典文件,加入到项目的根目录
paoding.dic.home=/paoding/dic

 

#seconds for dic modification detection


paoding.dic.detector.interval=60

---------------------------------------------------------------

paoding-knives.properties 文件配置

 

paoding.knife.class.letterKnife=net.paoding.analysis.knife.LetterKnife
paoding.knife.class.numberKnife=net.paoding.analysis.knife.NumberKnife
paoding.knife.class.cjkKnife=net.paoding.analysis.knife.CJKKnife

---------------------------------------------------------------

 

 

附上代码:

 

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;

import net.paoding.analysis.analyzer.PaodingAnalyzer;
import net.paoding.analysis.examples.gettingstarted.BoldFormatter;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleFragmenter;
import org.apache.lucene.search.highlight.TokenSources;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;

public class LuceneChinese {

 

//数据文件夹
private static final String DATA_DIR="C://unicode";

//索引存放文件夹
private static final String INDEX_DIR="C://dir";

//字段
private static final String FIELD_NAME="content";

public static void main(String [] args)throws Exception{


createIndex();
search("");
}
/**
* 创建索引
*/
public static void createIndex(){
System.out.println("-------------------建立索引开始-----------------------");
long timeStart=System.currentTimeMillis();
try{
// PaodingChineseAnalyzer实现Analyzer接口,继承PaodingAnalyze,重写tokenizer方法,实现过滤分词
Analyzer analyzer = new PaodingChineseAnalyzer(new File("E://stopwords.txt"));
IndexWriter writer = new IndexWriter(FSDirectory.open(new File(INDEX_DIR)),
analyzer,true, IndexWriter.MaxFieldLength.LIMITED);
// 根据指定的目录把该目录下所有txt文件索引起来
indexDoc(writer, new File(DATA_DIR));
// 优化, 可以提高搜索速度。
writer.optimize();
writer.close();
}catch (IOException e) {
// TODO: handle exception
e.printStackTrace();
}
long timeEnd=System.currentTimeMillis();
System.out.println("-------------------建立索引耗时: "+(timeEnd-timeStart)+" 毫秒-----------------------");
}
/**
* 搜索
* @param keyword
* @throws IOException
* @throws ParseException
*/
public static void search(String queryString)
throws IOException,ParseException{

//输入搜索关键字
if (queryString==null||queryString=="") {
System.out.print("Search for:");
InputStreamReader in=new InputStreamReader(System.in);
BufferedReader reader=new BufferedReader(in);
queryString=reader.readLine();
if(queryString==""){
System.exit(0);
}
}

long timeStart=System.currentTimeMillis();

//读取索引文件
Directory directory=FSDirectory.open(new File(INDEX_DIR));
//PaodingChineseAnalyzer实现Analyzer接口,继承PaodingAnalyzer
Analyzer analyzer = new PaodingChineseAnalyzer();
IndexReader reader = IndexReader.open(directory, true);
QueryParser parser = new QueryParser(FIELD_NAME, analyzer);
Query query = parser.parse(queryString);
//创建索引查询器
Searcher searcher = new IndexSearcher(directory);
query = query.rewrite(reader);
Hits hits = searcher.search(query);

//高亮显示标签,默认是<b></b>
//SimpleHTMLFormatter shf = new SimpleHTMLFormatter("<span style="color:red" mce_style="color:red">", "</span>");
BoldFormatter formatter = new BoldFormatter();
//构造高亮器,指定高亮的格式,指定查询计分器
Highlighter highlighter = new Highlighter(formatter, new QueryScorer(query));
//设置块划分器
highlighter.setTextFragmenter(new SimpleFragmenter(50));

System.out.println("共搜索到: "+hits.length()+" 条资源");
System.out.println("---------------------------------------------");
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
String path = hits.doc(i).get("path");
int maxNumFragmentsRequired = 5;
String fragmentSeparator = "...";
TermPositionVector tpv = (TermPositionVector) reader
.getTermFreqVector(hits.id(i), FIELD_NAME);
TokenStream tokenStream = TokenSources.getTokenStream(tpv);
String result = highlighter.getBestFragments(tokenStream, text,
maxNumFragmentsRequired, fragmentSeparator);
System.out.println("/n文件路径:" + path);
System.out.println("/n" + result);
}
reader.close();
System.out.println("共搜索到: "+hits.length()+" 条资源");
long timeEnd=System.currentTimeMillis();
System.out.println("-------------------查询耗时: "+(timeEnd-timeStart)+" 毫秒-----------------------");
}
/**
* 对指定的目录进行索引
*
* @param writer
* IndexWriter
* @param root
* 指定的目录
*/
private static void indexDoc(IndexWriter writer, File root) {
// 不去索引不能读的文件
if (root.canRead()) {
if (root.isDirectory()) {
File[] files = root.listFiles();
if (files.length != 0) {
for (int i = 0; i < files.length; i++) {
// 递归调用
indexDoc(writer, files[i]);
}
}
} else {
try {
// 文件的文本内容
InputStream in=new FileInputStream(root);
byte b[]=new byte[in.available()];
in.read(b);
String content = new String(b,"GBK");
// 创建一个lucene document
Document d = new Document();
// 把文件的文本内容添加进来 进行索引,保存
d.add(new Field(FIELD_NAME, content, Field.Store.YES,
Field.Index.TOKENIZED,Field.TermVector.WITH_POSITIONS_OFFSETS));
// 同时把path也加入进来,只存储,不索引
d.add(new Field("path", root.getAbsolutePath(),
Field.Store.YES, Field.Index.NOT_ANALYZED));
// 把document写入索引文件
writer.addDocument(d);
System.out.println("add file: " + root.getAbsolutePath());
} catch (FileNotFoundException e) {
System.out.println("file not found, ignored.");
e.printStackTrace();
} catch (IOException e) {

}
}
}
}

}

 

PaodingChineseAnalyzer 代码

 

 

import java.io.File;
import java.io.Reader;
import java.util.Set;

import net.paoding.analysis.analyzer.PaodingAnalyzer;
import net.paoding.analysis.analyzer.PaodingTokenizer;

import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WordlistLoader;

public class PaodingChineseAnalyzer extends PaodingAnalyzer{

private static String[] stopWords = {

"www","的","和","与","时","在",

"是","被","所","那","这","有",

"将","会","为","对","了","过",

"去"};
private Set stopSet;

public PaodingChineseAnalyzer() {
stopSet = StopFilter.makeStopSet(stopWords);
}
public PaodingChineseAnalyzer(String[] stopWords) {
stopSet = StopFilter.makeStopSet(stopWords);
}

//读取外部stopwords文件
public PaodingChineseAnalyzer(File stopwordsFile){
try{
stopSet=WordlistLoader.getWordSet(stopwordsFile);
}catch (Exception e) {
e.printStackTrace();
}
}

//过滤分词
public final TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new PaodingTokenizer(reader, getKnife(),createTokenCollector());

//加入过滤分词方法,lucene也提供了很多过滤分词方法,可以选择使用
result=new StopFilter(result, stopSet);
result = new LowerCaseFilter(result);
return result;
}
}

 

由于时间问题,说明和注释不是很清楚,如有什么疑问,大家可以留言讨论,相互学习。。

顶一下
(1)
100%
踩一下
(0)
0%
------分隔线----------------------------
发表评论 所有评论
你还没登录,请先登录后再来评论!
推荐内容
新知先觉