Clean text

.github/workflows/main.yml

@@ -0,0 +1,48 @@
+# This is a basic workflow to help you get started with Actions
+
+name: run_tests
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [ main ]
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: windows
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      name: Checkout repo
+      uses: actions/checkout@v3
+      # Runs a single command using the runners shell
+      name: Set Up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+       with:
+          python-version: ${{ matrix.python-version }}
+        # Display the Python version being use
+      name: Display Python version
+      run: python -c "import sys; print(sys.version)"
+        # Install the package using the setup.py
+      name: Run tests
+      run: python -m unittest tests/test_extract_dataframe.py
+
+
+
+
+
+
+
+
+
+
+

.travis.yml

@@ -19,3 +19,4 @@ cache:
 # Command to run tests, e.g. python setup.py test
 script:
   - python -m unittest tests.test_extract_dataframe
+  - python -m unittest tests.test_clean_tweets_dataframe

README.md

@@ -10,7 +10,7 @@
 6. Create a new branch called `make_unittest` for creating a new unit test for extract_dataframe.py code.
 7. After completing the unit test writing, merge  “make_unittest”  to main branch
 8. In all cases when you merge, make sure you first do Pull Request, review, then accept the merge.
-9. Setup Travis CI to your repository such that when you git push new code (or merge a branch) to the main branch, the unit test in tests/*.py runs automatically. 10. All tests should pass.
+9. Setup Github Actions CI to your repository such that when you git push new code (or merge a branch) to the main branch, the unit test in tests/*.py runs automatically. 10. All tests should pass.
 
 After Completing this Challenge, you would have explore  
 

clean_tweets_dataframe.py

@@ -0,0 +1,97 @@
+import pandas as pd
+
+
+class Clean_Tweets:
+    """
+    The PEP8 Standard AMAZING!!!
+    """
+    def __init__(self, df:pd.DataFrame):
+        self.df = df
+        print('Automation in Action...!!!')
+
+    def drop_unwanted_column(self, df:pd.DataFrame)->pd.DataFrame:
+        """
+        remove rows that has column names. This error originated from
+        the data collection stage.  
+        """
+        unwanted_rows = df[df['retweet_count'] == 'retweet_count' ].index
+        df.drop(unwanted_rows , inplace=True)
+        df = df[df['polarity'] != 'polarity']
+
+        return df
+    def drop_duplicate(self, df:pd.DataFrame)->pd.DataFrame:
+        """
+        drop duplicate rows
+        """
+
+        df=df.drop_duplicates(inplace = True)    # Drop duplicates
+
+
+        return df
+    def convert_to_datetime(self, df:pd.DataFrame)->pd.DataFrame:
+        """
+        convert column to datetime
+        """
+
+        df['created_at'] = pd.to_datetime(df['created_at'])
+        #tweets from 2021 onwards
+        df['created_at'] = df[df['created_at'] >= '2020-12-31' ]
+
+        return df
+
+    def convert_to_numbers(self, df:pd.DataFrame)->pd.DataFrame:
+        """
+        convert columns like polarity, subjectivity, retweet_count
+        favorite_count etc to numbers
+        """
+
+
+        df['polarity'] = pd.to_numeric(df['Polarity'])
+        df['subjectivity'] = pd.to_numeric(df['subjectivity'])
+        df['retweet_count'] = pd.to_numeric(df['retweet_count'])
+        df['favorite_count'] = pd.to_numeric(df['favorite_count'])
+        df['followers_count'] = pd.to_numeric(df['followers_count'])
+        df['friends_count'] = pd.to_numeric(df['friends_count'])
+
+
+
+        return df
+    def remove_non_english_tweets(self, df:pd.DataFrame)->pd.DataFrame:
+        """
+        remove non english tweets from lang
+        """
+        non_english = df[df['lang'] != 'en']
+        df = df.drop( non_english , inplace=True)
+
+        return df
+    def drop_nan(self, df:pd.DataFrame)->pd.DataFrame:
+        """
+        remove nan values
+        """
+        df = df.dropna()
+
+        return df
+    def reset_index(self, df:pd.DataFrame)->pd.DataFrame:
+        """
+        resetting the index after dropping values
+        """
+        df = df.reset_index(drop=True)
+
+        return df
+    def clean_df(self, df: pd.DataFrame):
+        df = self.drop_unwanted_column(df)
+        df = self.remove_non_english_tweets(df)
+        df = self.drop_duplicate(df)
+        df = self.convert_to_datetime(df)
+        df = self.convert_to_numbers(df)
+        df = self.drop_nan(df)
+        df = self.reset_index(df)
+
+        return df
+
+if __name__ == "__main__":
+    df=pd.read_csv('processed_tweet_data.csv')
+
+    data=Clean_Tweets(df)
+
+    cleaned_data=data.clean_df(df)

extract_dataframe.py

@@ -0,0 +1,199 @@
+import json
+import pandas as pd
+from textblob import TextBlob
+
+def read_json(json_file: str)->list:
+    """
+    json file reader to open and read json files into a list
+    Args:
+    -----
+    json_file: str - path of a json file
+
+    Returns
+    -------
+    length of the json file and a list of json
+    """
+
+    tweets_data = []
+    for tweets in open(json_file,'r'):
+        tweets_data.append(json.loads(tweets))
+
+
+    return len(tweets_data), tweets_data
+
+class TweetDfExtractor:
+    """
+    this function will parse tweets json into a pandas dataframe
+
+    Return
+    ------
+    dataframe
+    """
+    def __init__(self, tweets_list):
+
+        self.tweets_list = tweets_list
+
+    # an example function
+    def find_created_time(self)->list:
+        created_at=[]
+        for items in self.tweets_list:
+            #print(items)
+            #break;
+            created_at.append(items['created_at'])
+            #print(len(created_at)) #24625
+        return created_at
+
+
+    def find_source(self)->list:
+        source=[]
+        for items in self.tweets_list:
+            #print (items)
+            source.append(items['source'])
+
+        return source
+     def find_full_text(self)->list:
+
+        text=[]
+        for items in self.tweets_list:
+            text.append(items['text'])
+
+        return text
+
+
+
+    def find_sentiments(self, text)-> str:
+        polarity = [] # contains the polarity values from the sentiment analysis.
+        self.subjectivity = [] # contains the subjectivity values from the sentiment analysis.
+        for items in text:
+            self.subjectivity.append(TextBlob(items).sentiment.subjectivity)
+            polarity.append(TextBlob(items).sentiment.polarity)
+
+        return polarity, self.subjectivity
+
+    def find_lang(self)->list:
+        lang=[]
+        for items in self.tweets_list:
+            lang.append(items['lang'])
+
+        return lang
+
+    def find_favourite_count(self)->list:
+        favorites_count=[]
+        for items in self.tweets_list:
+            favorites_count.append(items['favorite_count'])
+
+        return favorites_count
+
+    def find_retweet_count(self)->list:
+        retweet_count=[]
+        for items in self.tweets_list:
+            retweet_count.append(items['retweet_count'])
+
+        return retweet_count
+
+    def find_screen_name(self)->list:
+        screen_name=[]
+        for items in self.tweets_list:
+            screen_name.append(items['user']['screen_name'])
+
+        return screen_name
+
+    def find_followers_count(self)->list:
+        followers_count=[]
+        for items in self.tweets_list:
+            followers_count.append(items['user']['followers_count'])
+
+        return followers_count
+
+    def find_friends_count(self)->list:
+        friends_count=[]
+        for items in self.tweets_list:
+            friends_count.append(items['user']['friends_count'])
+
+        return friends_count
+
+
+    def find_location(self)->list:
+        """
+        a function that extracts the location.
+        returns list of locations
+        """
+        location = [x.get('retweeted_status', {}).get('user', {}).get('location', None) for x in self.tweets_list]
+        return location
+
+    def is_sensitive(self)->list:
+        try:
+            is_sensitive = [x['possibly_sensitive'] for x in self.tweets_list]
+        except KeyError:
+            is_sensitive = ''
+
+
+        return is_sensitive
+
+    def find_hashtags(self)->list:
+        hashtags = []
+        for items in self.tweets_list:
+            hashtags.append(items['entities']['hashtags'])
+        return hashtags
+    def find_mentions(self)->list:
+        mentions=[ ]
+        for items in self.tweets_list:
+            mentions.append(items['entities']['user_mentions'])
+        return mentions
+
+
+    def find_statuses_count(self)->list:
+        status_count
+        status_count = self.tweets_list["status_count"]
+        return status_count
+
+
+
+    def get_tweet_df (self, save=True)->pd.DataFrame:
+        """required column to be generated you should be creative and add more features"""
+
+        '''columns = ['created_at', 'source', 'original_text','polarity','subjectivity', 'lang', 'favorite_count', 'retweet_count', 
+            'original_author', 'followers_count','friends_count','possibly_sensitive', 'hashtags', 'user_mentions', 'place']'''
+        columns = ['created_at', 'source', 'original_text','polarity','subjectivity', 'lang', 'favorite_count', 'retweet_count', 
+            'original_author', 'followers_count','friends_count', 'hashtags', 'user_mentions','place']
+
+        created_at = self.find_created_time()
+        source = self.find_source()
+        text = self.find_full_text()
+        polarity, subjectivity = self.find_sentiments(text)
+        lang = self.find_lang()
+        fav_count = self.find_favourite_count()
+        retweet_count = self.find_retweet_count()
+        screen_name = self.find_screen_name()
+        follower_count = self.find_followers_count()
+        friends_count = self.find_friends_count()
+
+        hashtags = self.find_hashtags()
+        mentions = self.find_mentions()
+        location = self.find_location()
+        data = zip(created_at, source, text, polarity, subjectivity, lang, fav_count, retweet_count, screen_name, follower_count,
+                        friends_count, hashtags, mentions,location)
+
+        #this creates a list of tuples
+        df = pd.DataFrame(data=data, columns=columns)
+
+        if save:
+            df.to_csv('processed_tweet_data.csv', index=False)
+            print('File Successfully Saved.!!!')
+
+
+        return df
+
+if __name__ == "__main__":
+    # required column to be generated you should be creative and add more features
+    columns = ['created_at', 'source', 'original_text','clean_text', 'sentiment','polarity','subjectivity', 'lang', 'favorite_count',                   'retweet_count','original_author', 'screen_count', 'followers_count','friends_count','possibly_sensitive', 'hashtags',                          'user_mentions', 'place', 'place_coord_boundaries']
+    tweet_num ,tweet_list = read_json("Economic_Twitter_Data.json")
+    tweet = TweetDfExtractor(tweet_list)
+    tweet_df = tweet.get_tweet_df() 
+
+    # use all defined functions to generate a dataframe with the specified columns above
+
+
+
+
+