Query-Likelihood-Retrieval-Model/Indexer.py at main · riya-joshi-401/Query-Likelihood-Retrieval-Model · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# Importing necessary libraries
import glob
import os
import sys

# corpus directory without stopped and stemmed data
DIR_CORPUS='CorpusGeneration/TokenizedCorpus'
# corpus directory with stopped data
DIR_CORPUS_WITH_STOPPING='CorpusGeneration/TokenizedCorpusWithStopping'
# corpus directory with stemmed data
DIR_CORPUS_WITH_STEMMING='CorpusGeneration/TokenizedCorpusWithStemming'
# directory for index text files
DIR_INDEX_TEXT_FILES='Indexing/IndexTextFiles/'


# method to create index
#  TERM:{docId:tf}
def buildIndex(corpusDirectory):
    invertedList={}
    noOfTokensInDoc={}
    path=os.path.join(corpusDirectory,r"*.txt")
    tokenFiles=glob.glob(path)
    for fName in tokenFiles:
        f=open(fName,'r')
        listOfTokens=f.read().split()
        docName = fName.split('/')[-1].split('.')[0]
        print(docName)
        noOfTokensInDoc[docName]=len(listOfTokens)
        for token in listOfTokens:
            if not token in invertedList:
                invertedList[token]={docName:1}
            else:
                doc=invertedList[token]
                if not docName in doc:
                    doc[docName]=1
                else:
                    doc[docName]+=1

    print("No. of indexed terms: ",str(len(invertedList)))
    return (invertedList,noOfTokensInDoc)


# create text files for each unigram index
def writeIndexToTextFile(index,mode):
    if not os.path.exists(DIR_INDEX_TEXT_FILES):
        os.makedirs(DIR_INDEX_TEXT_FILES)
    fName=DIR_INDEX_TEXT_FILES+'unigram-'+mode+'-index.txt'
    if os.path.exists(fName):
        os.remove(fName)
    fileIndex=open(fName,'w')
    # fileIndex.write('##########################################################################################\n')
    # fileIndex.write('################################ Unigram Inverted Index ##################################\n')
    # fileIndex.write('##########################################################################################\n\n\n')
    fileIndex.write("{")
    for term in index:
        fileIndex.write("'"+term+"' : ")
        docDic=index[term]
        s=[]
        for doc in docDic:
            tup=()+(str(doc),)+(str(docDic[doc]),)
            s.append(tup)
        fileIndex.write(str(s)+",")
    fileIndex.write("}")
    fileIndex.close()


def generateNoOfTermsPerDocFile(noOfTokensPerDoc,mode):
    if not os.path.exists(DIR_INDEX_TEXT_FILES):
        os.makedirs(DIR_INDEX_TEXT_FILES)
    fName=DIR_INDEX_TEXT_FILES+'NoTokensPerDoc-'+mode+'.txt'
    if os.path.exists(fName):
        os.remove(fName)
    fileIndex=open(fName,'w')
    # fileIndex.write('##########################################################################################\n')
    # fileIndex.write('#################################### Tokens Per Document #################################\n')
    # fileIndex.write('##########################################################################################\n\n\n')
    fileIndex.write("{")
    for doc in noOfTokensPerDoc:
        fileIndex.write("'"+doc+"':  "+str(noOfTokensPerDoc[doc])+",")
    fileIndex.write("}")
    fileIndex.close()


def selectTheCorpusForIndexing1():
    while True:
        print("\nSelect the corpus for indexing:")
        print("Enter 1 No Stopping and Stemming")
        print("Enter 2 With Stopping")
        print("Enter 3 With Stemming")
        print("Enter 4 to exit!!")
        x=int(input("Enter choice: "))
        if x==1:
            corpusDirectory=DIR_CORPUS
            mode="no_stopping_or_stemming"
        elif x==2:
            corpusDirectory=DIR_CORPUS_WITH_STOPPING
            mode="withStopping"
        elif x==3:
            corpusDirectory=DIR_CORPUS_WITH_STEMMING
            mode="withStemming"
        else:
            break

        print('Generating Unigram Index.......')
        unigramIndex,noOfTokensInDoc=buildIndex(corpusDirectory)
        generateNoOfTermsPerDocFile(noOfTokensInDoc,mode)
        writeIndexToTextFile(unigramIndex,mode)

if __name__=='__main__':
    selectTheCorpusForIndexing1()