Phonetic-Encoding/mainHemming.py at master · Adalsteinnjons/Phonetic-Encoding · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from hemmingDistance import HemmingDistance

# Pleaser remove emptyline at the end of the .txt file before proceeding

def extractWords(path):
    text = open(path, 'r')
    text = text.read()
    text = [s.strip() for s in text.split('\n')]
    return text

def intersection(list1,list2):
    list3 = list(set(list1) & set(list2))
    return list3

def cleanDuplicate(names):
    return list(set(names))

def trueRate(names,controlList):
    return (len(intersection(names,controlList))/len(controlList)) * 100

corruptPath = './cleaningDataset/corruptedNames.txt'
femalePath = './cleaningDataset/femaleFirstnames.txt'
malePath = './cleaningDataset/maleFirstnames.txt'
lastNamesPath = './cleaningDataset/lastnames.txt'
generatedPath = './cleaningDataset/generatedNames.txt'

corruptedNames = extractWords(corruptPath)
femaleFirstNamesDict = extractWords(femalePath)
maleFirstNamesDict = extractWords(malePath)
lastNamesDict = extractWords(lastNamesPath)
generatedNames = extractWords(generatedPath)

cleanedCorruptedNames = cleanDuplicate(corruptedNames)
print('=============================================================')
print('HEMMING DISTANCE')
print('=============================================================')
print('True positive rate:')
print('Before the cleaning: %.2f'% trueRate(cleanedCorruptedNames,generatedNames), '%')


hd = HemmingDistance()

typoFixed = []
counter = 0
for name in cleanedCorruptedNames:
    counter += 1
    if (counter%1000 == 0):
        print(counter, ' names processed')
    typoFixed.append(hd.fixTypo(name,lastNamesDict))
#
# f = open("./preprocessed/noTypoLastNames.txt", "w+")
# for name in typoFixed:
#     if typoFixed[-1] == name:
#         f.write(name)
#     else:
#         f.write(name + '\n')

# typoFixed = extractWords('./preprocessed/noTypoLastNames.txt')
typoFixed = cleanDuplicate(typoFixed)
# print([value.split(" ") for value in typoFixed if len(value.split(" "))== 1 ])
print('After cleaning the lastnames: %.2f' % trueRate(typoFixed,generatedNames), '%')


maleFirstNamesDict.extend(femaleFirstNamesDict)

typoFixedFirstNames = []
counter = 0
for name in typoFixed:
    counter += 1
    if (counter%1000 == 0):
        print(counter,' names processed')
    typoFixedFirstNames.append(hd.fixTypoFirstNames(name,maleFirstNamesDict))
#
# f = open("./preprocessed/hemmingFinal.txt", "w+")
# for name in typoFixedFirstNames:
#     if typoFixedFirstNames[-1] == name:
#         f.write(name)
#     else:
#         f.write(name + '\n')

# typoFixedFirstNames = extractWords("./preprocessed/hemmingFinal.txt")
typoFixedFirstNames = cleanDuplicate(typoFixedFirstNames)
print('After cleaning the firstnames: %.2f' % trueRate(typoFixedFirstNames,generatedNames), '%')
print('=============================================================')

print('Number of eliminated Duplicates: ',len(corruptedNames)-len(typoFixedFirstNames), 'Names')