Fake-news-detection/TrainTestSplit.py at master · ayush29/Fake-news-detection · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 22 15:54:23 2018

@author: Ayush jain
"""
import pandas as pd
from urllib.parse import urlparse
import Preprocess
#import tldextract
print("traintestsplit.py")
def ClaimsTrainTestSplit():
    claimFrame = pd.read_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/claimCredibility.csv')
    reviewsFrame = pd.read_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/claimReviews.csv')
    claimTrainFrame = claimFrame.iloc[:40000][:]
    claimTestFrame = claimFrame.iloc[40000:][:]
    reviewsTrainFrame = reviewsFrame.loc[reviewsFrame['claimId'].isin(claimTrainFrame['claimId'])]
    reviewsTestFrame = reviewsFrame.loc[reviewsFrame['claimId'].isin(claimTestFrame['claimId'])]
    claimTrainFrame.to_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Train/claimCredibility.csv')
    claimTestFrame.to_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Test/claimCredibility.csv')
    reviewsTrainFrame.to_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Train/claimReviews.csv')
    reviewsTestFrame.to_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Test/claimReviews.csv')

#ClaimsTrainTestSplit()

def getClaimTextFromLink(claimLink):
    #if(not isinstance(claimLink,str)):
    #    claimLink = claimLink.iloc[0]
    claims = pd.read_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/claims.csv',index_col=0)
    return claims.loc[claimLink,:].tolist()[0]

def getReviewsFromClaimId(claimId,reviewFile):
    reviews = pd.read_csv(reviewFile)
    reviews = reviews.loc[reviews['claimId'].isin([claimId])]
    #print(reviews)
    reviews = reviews.loc[:,['Review','Reviewer']]
    reviewlist = []
    reviewerslist = []
    for i in reviews.index:

        review = reviews.loc[i,'Review']
        reviewer = reviews.loc[i,'Reviewer']
        reviewlist.append(review)
        reviewerslist.append(reviewer)
    return reviewlist,reviewerslist

def getClaimSourceAttrFromClaimLink(claimLink):
    #if( not isinstance(claimLink,str)):
    #    claimLink = claimLink.iloc[0]
    domain = urlparse(claimLink).netloc
    #print(domain)
    #domain = tldextract.extract(claimLink).domain
    sourceFrame = pd.read_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/claim_sources.csv',names=['source', 'a1','a2','a3','a4','a5','a6','a7'], header=None)
    sourceFrame = sourceFrame.loc[sourceFrame['source'].str.contains(domain)]
    sourceFrame = sourceFrame.drop_duplicates(subset=['source'], keep='last') #removing duplicate rows
    return sourceFrame.loc[:,['a1','a2','a3','a4','a5','a6','a7']].values

def getReviewerAttrFromName(name)  :
    name = name.strip(' ').lower()
    name = name.replace(' ','-')
    reviewer = pd.read_csv('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/members.csv',index_col=0,names=['f1','f2','f3','f4','f5','f6','f7','f8','f9','f10','f11','f12'])
    reviewer = reviewer[~reviewer.index.duplicated(keep='last')] #removing duplicate rows
    return reviewer.loc[name,:].values

def getBatch(claimFile,reviewFile,batchSize):
    claimFrame = pd.read_csv(claimFile,index_col=0)
    claimFrame = claimFrame[~claimFrame.index.duplicated(keep='first')] #removing duplicate rows
    X =[]
    Y = []
    counter=0
    while True:
        for claimId in claimFrame.index.tolist():
            # print('caught!\n')
            try:
                reviews,reviewers = getReviewsFromClaimId(claimId,reviewFile)
                claimSourceEmbedding = getClaimSourceAttrFromClaimLink(claimFrame.loc[claimId,'claimLink'])
                credibility = claimFrame.loc[claimId,'credibility']
            #if(isinstance(credibility,pd.Series)): #above statement may return a series if duplicate entries are there in claimframe
            #    credibility = credibility.iloc[0]
                claimText = getClaimTextFromLink(claimFrame.loc[claimId,'claimLink'])
            except:
                continue
            claimEmbeddings = Preprocess.getUniversalSentenceEncoding(claimText)
            print("no. of reviews"+str(len(reviews)))
            #print(reviews)
            if len(reviews) > 0:
                for i in range(len(reviews)):
                    tokens = Preprocess.tokenizer(reviews[i])
                    #articleTerms = tokens
                    articleTermEmbeddings = Preprocess.getContextualisedWordEmbeddings(tokens)
                    reviewerEmbedding = getReviewerAttrFromName(reviewers[i])
                    x = {'claimId':claimId,'claim' : claimEmbeddings, 'article': articleTermEmbeddings,'claimSource':claimSourceEmbedding,'articleSource':reviewerEmbedding }
                    y = credibility
                    X.append(x)
                    Y.append(y)
                    counter = counter +1
                    if(counter == batchSize):
                       yield X,Y
                       X=[]
                       Y=[]
                       counter = 0

    #while True:

#getClaimTextFromLink("b'http://www.npr.org/templates/story/story.php?storyId=121529261'")

#TrainDATa
#XTrain,YTrain = next(getBatch('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Train/claimCredibility.csv','/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Train/claimReviews.csv',3000))
#print(X)
#print(Y)
#ClaimsTrainTestSplit()

#with open('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Train/XTrain.pickle', 'wb') as handle:
#    pickle.dump(XTrain, handle, protocol=pickle.HIGHEST_PROTOCOL)
#with open('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Train/YTrain.pickle', 'wb') as handle:
#	pickle.dump(YTrain,handle, protocol=pickle.HIGHEST_PROTOCOL)

##Test Data##
#XTest,YTest = next(getBatch('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Test/claimCredibility.csv','/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Test/claimReviews.csv',1000))

#print(X)
#print(Y)
#ClaimsTrainTestSplit()

#with open('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Test/XTest.pickle', 'wb') as handle:
#    pickle.dump(XTest, handle, protocol=pickle.HIGHEST_PROTOCOL)
#with open('/media/sdb/sanjay/IR/debunking-fake-news/newstrust_final/Test/YTest.pickle', 'wb') as handle:
#        pickle.dump(YTest,handle, protocol=pickle.HIGHEST_PROTOCOL)