分析器与分词 #

输入文本
    ↓
CharFilter（字符过滤器）
    ↓
Tokenizer（分词器）
    ↓
TokenFilter（词元过滤器）
    ↓
Token流

<fieldType name="text_standard" class="solr.TextField">
  <analyzer>
    <tokenizer class="solr.StandardTokenizerFactory"/>
  </analyzer>
</fieldType>

"Solr is a search engine" → ["Solr", "is", "a", "search", "engine"]

<fieldType name="text_whitespace" class="solr.TextField">
  <analyzer>
    <tokenizer class="solr.WhitespaceTokenizerFactory"/>
  </analyzer>
</fieldType>

"Solr Search Engine" → ["Solr", "Search", "Engine"]

<fieldType name="text_keyword" class="solr.TextField">
  <analyzer>
    <tokenizer class="solr.KeywordTokenizerFactory"/>
  </analyzer>
</fieldType>

"Solr Search Engine" → ["Solr Search Engine"]

<fieldType name="text_pattern" class="solr.TextField">
  <analyzer>
    <tokenizer class="solr.PatternTokenizerFactory" pattern="[,;]"/>
  </analyzer>
</fieldType>

"a,b;c" → ["a", "b", "c"]

<fieldType name="text_path" class="solr.TextField">
  <analyzer>
    <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/"/>
  </analyzer>
</fieldType>

"/a/b/c" → ["/a", "/a/b", "/a/b/c"]

<filter class="solr.LowerCaseFilterFactory"/>

"Solr" → "solr"

<filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>

a
an
and
are
as
at
be
but
by

<filter class="solr.SynonymGraphFilterFactory" 
        synonyms="synonyms.txt" 
        ignoreCase="true" 
        expand="true"/>

iphone,苹果手机,Apple手机
ipad,苹果平板,Apple平板

<!-- 英文词干 -->
<filter class="solr.PorterStemFilterFactory"/>

<!-- 雪球词干 -->
<filter class="solr.SnowballPorterFilterFactory" language="English"/>

"running" → "run"
"jumped" → "jump"

<filter class="solr.ASCIIFoldingFilterFactory"/>

"café" → "cafe"
"naïve" → "naive"

<filter class="solr.LengthFilterFactory" min="2" max="20"/>

<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>

<charFilter class="solr.MappingCharFilterFactory" mapping="mapping.txt"/>

" => "
' => '

<charFilter class="solr.HTMLStripCharFilterFactory"/>

"<p>Solr</p>" → "Solr"

<charFilter class="solr.PatternReplaceCharFilterFactory" 
            pattern="[-]" replacement=""/>

<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
  <analyzer type="index">
    <tokenizer class="solr.StandardTokenizerFactory"/>
    <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
    <filter class="solr.LowerCaseFilterFactory"/>
    <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
  </analyzer>
  <analyzer type="query">
    <tokenizer class="solr.StandardTokenizerFactory"/>
    <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
    <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
    <filter class="solr.LowerCaseFilterFactory"/>
    <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
  </analyzer>
</fieldType>

<fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
  <analyzer type="index">
    <charFilter class="solr.HTMLStripCharFilterFactory"/>
    <tokenizer class="solr.StandardTokenizerFactory"/>
    <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
    <filter class="solr.LowerCaseFilterFactory"/>
    <filter class="solr.EnglishPossessiveFilterFactory"/>
    <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
    <filter class="solr.PorterStemFilterFactory"/>
  </analyzer>
  <analyzer type="query">
    <charFilter class="solr.HTMLStripCharFilterFactory"/>
    <tokenizer class="solr.StandardTokenizerFactory"/>
    <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
    <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
    <filter class="solr.LowerCaseFilterFactory"/>
    <filter class="solr.EnglishPossessiveFilterFactory"/>
    <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
    <filter class="solr.PorterStemFilterFactory"/>
  </analyzer>
</fieldType>

<fieldType name="text_cn" class="solr.TextField">
  <analyzer type="index">
    <tokenizer class="org.wltea.analyzer.lucene.IKTokenizerFactory" useSmart="false"/>
    <filter class="solr.LowerCaseFilterFactory"/>
  </analyzer>
  <analyzer type="query">
    <tokenizer class="org.wltea.analyzer.lucene.IKTokenizerFactory" useSmart="true"/>
    <filter class="solr.LowerCaseFilterFactory"/>
  </analyzer>
</fieldType>

curl "http://localhost:8983/solr/mycore/analysis/field" \
  -d "fieldtype=text_general" \
  -d "fieldvalue=Solr is a search engine"

curl "http://localhost:8983/solr/mycore/analysis/field" \
  -d "field=title" \
  -d "fieldvalue=Solr is a search engine"

curl "http://localhost:8983/solr/mycore/analysis/document" \
  -d "analysis.fieldvalue=Solr is a search engine"

{
  "analysis": {
    "field_types": {
      "text_general": {
        "index": [
          {
            "text": "Solr is a search engine",
            "tokens": [
              {"text": "solr", "start": 0, "end": 4},
              {"text": "is", "start": 5, "end": 7},
              {"text": "a", "start": 8, "end": 9},
              {"text": "search", "start": 10, "end": 16},
              {"text": "engine", "start": 17, "end": 23}
            ]
          }
        ]
      }
    }
  }
}

# 下载IK分词器
wget https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.17.0/elasticsearch-analysis-ik-7.17.0.zip

# 解压到Solr的lib目录
unzip elasticsearch-analysis-ik-7.17.0.zip -d server/solr-webapp/webapp/WEB-INF/lib/

<fieldType name="text_ik" class="solr.TextField">
  <analyzer type="index">
    <tokenizer class="org.wltea.analyzer.lucene.IKTokenizerFactory" useSmart="false"/>
  </analyzer>
  <analyzer type="query">
    <tokenizer class="org.wltea.analyzer.lucene.IKTokenizerFactory" useSmart="true"/>
  </analyzer>
</fieldType>

<fieldType name="text_smartcn" class="solr.TextField">
  <analyzer>
    <tokenizer class="solr.SmartChineseSentenceTokenizerFactory"/>
    <filter class="solr.SmartChineseStopTokenFilterFactory"/>
  </analyzer>
</fieldType>

# custom/ext.dic
搜索引擎
分布式搜索
全文检索

# 同义词配置
# 格式1：等价同义词
iphone,苹果手机,Apple手机

# 格式2：单向同义词
苹果手机 => iphone

<filter class="solr.SynonymGraphFilterFactory" 
        synonyms="synonyms.txt" 
        ignoreCase="true" 
        expand="true"/>

搜索"苹果手机" → 匹配"iphone"、"苹果手机"、"Apple手机"

<filter class="solr.SynonymGraphFilterFactory" 
        synonyms="synonyms.txt" 
        ignoreCase="true" 
        expand="false"/>

搜索"苹果手机" → 只匹配"iphone"

# 英文停用词
a
an
and
are
as
at
be
but
by
for
if
in
into
is
it
no
not
of
on
or
such
that
the
their
then
there
these
they
this
to
was
will
with

<filter class="solr.StopFilterFactory" 
        ignoreCase="true" 
        words="stopwords.txt"/>