mirror of
https://github.com/mindoc-org/mindoc.git
synced 2026-01-23 05:12:14 +08:00
41 lines
681 B
Go
41 lines
681 B
Go
package engine
|
||
|
||
import (
|
||
"bufio"
|
||
"log"
|
||
"os"
|
||
)
|
||
|
||
type StopTokens struct {
|
||
stopTokens map[string]bool
|
||
}
|
||
|
||
// 从stopTokenFile中读入停用词，一个词一行
|
||
// 文档索引建立时会跳过这些停用词
|
||
func (st *StopTokens) Init(stopTokenFile string) {
|
||
st.stopTokens = make(map[string]bool)
|
||
if stopTokenFile == "" {
|
||
return
|
||
}
|
||
|
||
file, err := os.Open(stopTokenFile)
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
defer file.Close()
|
||
|
||
scanner := bufio.NewScanner(file)
|
||
for scanner.Scan() {
|
||
text := scanner.Text()
|
||
if text != "" {
|
||
st.stopTokens[text] = true
|
||
}
|
||
}
|
||
|
||
}
|
||
|
||
func (st *StopTokens) IsStopToken(token string) bool {
|
||
_, found := st.stopTokens[token]
|
||
return found
|
||
}
|