9 changed files with 283 additions and 0 deletions
@ -0,0 +1,83 @@ |
|||||
|
package textcomparison |
||||
|
|
||||
|
import ( |
||||
|
"sync" |
||||
|
|
||||
|
"github.com/yanyiwu/gojieba" |
||||
|
) |
||||
|
|
||||
|
type GoJieba struct { |
||||
|
C *gojieba.Jieba |
||||
|
} |
||||
|
|
||||
|
var GJB *GoJieba |
||||
|
var one sync.Once |
||||
|
|
||||
|
func NewGoJieba() *GoJieba { |
||||
|
one.Do(func() { |
||||
|
GJB = &GoJieba{ |
||||
|
C: gojieba.NewJieba(), |
||||
|
//equals with x := NewJieba(DICT_PATH, HMM_PATH, USER_DICT_PATH)
|
||||
|
} |
||||
|
}) |
||||
|
return GJB |
||||
|
} |
||||
|
|
||||
|
func (this *GoJieba) Close() { |
||||
|
this.C.Free() |
||||
|
} |
||||
|
|
||||
|
func (this *GoJieba) AddWords(words []string) { |
||||
|
for _, word := range words { |
||||
|
this.C.AddWord(word) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (this *GoJieba) JiebaCut(rawStr string, useHmm bool, cutAll bool) (words []string) { |
||||
|
if cutAll { |
||||
|
words = jiebaCutAll(this.C, &rawStr) |
||||
|
} else { |
||||
|
words = jiebaCut(this.C, &rawStr, useHmm) |
||||
|
} |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (this *GoJieba) JiebaCutWithFrequency(rawStr string, useHmm bool, cutAll bool) (wordsFreqs map[string]int) { |
||||
|
wordsFreqs = make(map[string]int) |
||||
|
if cutAll { |
||||
|
words := jiebaCutAll(this.C, &rawStr) |
||||
|
for _, word := range words { |
||||
|
freq := wordsFreqs[word] |
||||
|
wordsFreqs[word] = freq + 1 |
||||
|
} |
||||
|
} else { |
||||
|
words := jiebaCut(this.C, &rawStr, useHmm) |
||||
|
for _, word := range words { |
||||
|
freq := wordsFreqs[word] |
||||
|
wordsFreqs[word] = freq + 1 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (this *GoJieba) JiebaCutForSearch(rawStr string, useHmm bool) { |
||||
|
jiebaCut4Search(this.C, &rawStr, useHmm) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func jiebaCutAll(x *gojieba.Jieba, rawStr *string) (words []string) { |
||||
|
words = x.CutAll(*rawStr) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func jiebaCut(x *gojieba.Jieba, rawStr *string, useHmm bool) (words []string) { |
||||
|
words = x.Cut(*rawStr, useHmm) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func jiebaCut4Search(x *gojieba.Jieba, rawStr *string, useHmm bool) (words []string) { |
||||
|
words = x.CutForSearch(*rawStr, useHmm) |
||||
|
return |
||||
|
} |
||||
@ -0,0 +1,109 @@ |
|||||
|
package textcomparison |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"hash/fnv" |
||||
|
"strings" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
SIMILAR_DISTANCE = 3 |
||||
|
) |
||||
|
|
||||
|
type WordWeight struct { |
||||
|
Word string |
||||
|
Weight float64 |
||||
|
} |
||||
|
|
||||
|
func SimHashSimilar(srcWordWeighs, dstWordWeights []WordWeight) (distance int, err error) { |
||||
|
|
||||
|
srcFingerPrint, err := simhashFingerPrint(srcWordWeighs) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
fmt.Println("srcFingerPrint: ", srcFingerPrint) |
||||
|
dstFingerPrint, err := simhashFingerPrint(dstWordWeights) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
fmt.Println("dstFingerPrint: ", dstFingerPrint) |
||||
|
|
||||
|
distance = hammingDistance(srcFingerPrint, dstFingerPrint) |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func simhashFingerPrint(wordWeights []WordWeight) (fingerPrint []string, err error) { |
||||
|
binaryWeights := make([]float64, 32) |
||||
|
for _, ww := range wordWeights { |
||||
|
bitHash := strHashBitCode(ww.Word) |
||||
|
weights := calcWithWeight(bitHash, ww.Weight) //binary每个元素与weight的乘积结果数组
|
||||
|
binaryWeights, err = sliceInnerPlus(binaryWeights, weights) |
||||
|
//fmt.Printf("ww.Word:%v, bitHash:%v, ww.Weight:%v, binaryWeights: %v\n", ww.Word,bitHash, ww.Weight, binaryWeights)
|
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
fingerPrint = make([]string, 0) |
||||
|
for _, b := range binaryWeights { |
||||
|
if b > 0 { // bit 1
|
||||
|
fingerPrint = append(fingerPrint, "1") |
||||
|
} else { // bit 0
|
||||
|
fingerPrint = append(fingerPrint, "0") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func strHashBitCode(str string) string { |
||||
|
h := fnv.New32a() |
||||
|
h.Write([]byte(str)) |
||||
|
b := int64(h.Sum32()) |
||||
|
return fmt.Sprintf("%032b", b) |
||||
|
} |
||||
|
|
||||
|
func calcWithWeight(bitHash string, weight float64) []float64 { |
||||
|
bitHashs := strings.Split(bitHash, "") |
||||
|
binarys := make([]float64, 0) |
||||
|
|
||||
|
for _, bit := range bitHashs { |
||||
|
if bit == "0" { |
||||
|
binarys = append(binarys, float64(-1)*weight) |
||||
|
} else { |
||||
|
binarys = append(binarys, float64(weight)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return binarys |
||||
|
} |
||||
|
|
||||
|
func sliceInnerPlus(arr1, arr2 []float64) (dstArr []float64, err error) { |
||||
|
dstArr = make([]float64, len(arr1), len(arr1)) |
||||
|
|
||||
|
if arr1 == nil || arr2 == nil { |
||||
|
err = fmt.Errorf("sliceInnerPlus array nil") |
||||
|
return |
||||
|
} |
||||
|
if len(arr1) != len(arr2) { |
||||
|
err = fmt.Errorf("sliceInnerPlus array Length NOT match, %v != %v", len(arr1), len(arr2)) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
for i, v1 := range arr1 { |
||||
|
dstArr[i] = v1 + arr2[i] |
||||
|
} |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func hammingDistance(arr1, arr2 []string) int { |
||||
|
count := 0 |
||||
|
for i, v1 := range arr1 { |
||||
|
if v1 != arr2[i] { |
||||
|
count++ |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return count |
||||
|
} |
||||
@ -0,0 +1,30 @@ |
|||||
|
package textcomparison |
||||
|
|
||||
|
import ( |
||||
|
"regexp" |
||||
|
"strings" |
||||
|
) |
||||
|
|
||||
|
func RemoveHtml(src string) string { |
||||
|
//将HTML标签全转换成小写
|
||||
|
re, _ := regexp.Compile(`\\<[\\S\\s]+?\\>`) |
||||
|
src = re.ReplaceAllStringFunc(src, strings.ToLower) |
||||
|
|
||||
|
//去除STYLE
|
||||
|
re, _ = regexp.Compile(`\\<style[\\S\\s]+?\\</style\\>`) |
||||
|
src = re.ReplaceAllString(src, "") |
||||
|
|
||||
|
//去除SCRIPT
|
||||
|
re, _ = regexp.Compile(`\\<script[\\S\\s]+?\\</script\\>`) |
||||
|
src = re.ReplaceAllString(src, "") |
||||
|
|
||||
|
//去除所有尖括号内的HTML代码，并换成换行符
|
||||
|
re, _ = regexp.Compile(`\\<[\\S\\s]+?\\>`) |
||||
|
src = re.ReplaceAllString(src, "\n") |
||||
|
|
||||
|
//去除连续的换行符
|
||||
|
re, _ = regexp.Compile(`\\s{2,}`) |
||||
|
src = re.ReplaceAllString(src, "\n") |
||||
|
|
||||
|
return src |
||||
|
} |
||||
Loading…
Reference in new issue