Search_Engine_Scraper_2024/3_Social_Media_detector_2025.py at main · AndreM92/Search_Engine_Scraper_2024 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

import os
func_import = r"C:\Users\andre\Documents\Python\Web_Crawler\Search_Engine_Scraper_2024"
os.chdir(func_import)
from search_crawler_functions import *
from search_crawler_credentials import *
chromedriver_path = r"C:\Users\andre\Documents\Python\chromedriver-win64\chromedriver.exe"
startpage = 'https://www.google.de'

from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
import re
from datetime import datetime, timedelta
import time

folder_name = "SMP Energieanbieter 2026"
file_name = "Auswahl_SMP Energieanbieter 2026_20260507"
file_path = r"C:\Users\andre\OneDrive\Desktop/" + folder_name
source_file = file_name + ".xlsx"
branch_keywords = ['strom', 'gas', 'nachhaltig', 'versorgung', 'effizient', 'erneuerbar', 'solar', 'wärme', 'sparen', 'tarif', 'klima', 'netz']
########################################################################################################################

def scrape_page(driver, startpage, row, col_list, platform):
    comp_keywords, company, website, web_address2 = get_company_keywords(row, col_list)
    account = extract_text(row[platform])
    if len(account) > 10:
        if '?locale' in account:
            account = account.split('?locale')[0]
        result_row = [company, website, account, '', 'account from website']
        return result_row

    keyword = company + ' ' + platform
    try:
        search_url = search_for(driver, startpage, keyword)
    except:
        input('Press ENTER after solving the captcha')
        search_url = search_for(driver, startpage, keyword)
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    pagetext = get_visible_text(soup)
    if search_url == startpage or '/sorry' in search_url or 'CAPTCHA' in pagetext or '/consent' in search_url:
        input('Press ENTER after solving the captcha')
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        pagetext = get_visible_text(soup)
    if '403' in pagetext[:100] or 'That’s an error' in pagetext or 'not have permission' in pagetext:
        result_row = [company, website, account, '', 'Error: ' + pagetext]
        return result_row

    search_results = get_search_results(soup)
    account_list = rank_sm_accounts(platform, comp_keywords, branch_keywords, search_results)
    account, account2, account_list = get_accounts(account_list)
    result_row = [company, website, account, account2, account_list]
    return result_row

#######################################################################################################################
new_table = []
start_ID = 0
platform = 'LinkedIn'

if __name__ == '__main__':
    os.chdir(file_path)
    df_source = pd.read_excel(source_file)
    col_list = list(df_source.columns)

    # Start the selenium chromedriver
    driver = start_browser(webdriver, Service, chromedriver_path, headless=False, muted=True)
    go_to_page(driver, startpage)

    for ID, row in df_source.iterrows():
        if 'ID' in col_list:
            ID = row['ID']
        if ID < start_ID:                      #If you want to skip some rows
            continue
#        start_ID = ID + 1

        result_row = scrape_page(driver, startpage, row, col_list, platform)
        if 'Error' in result_row[-1]:
            input('Press ENTER after solving the issue')
        full_row = [ID] + result_row
        print(full_row)
        new_table.append(full_row)


    # Dataframe
    header = ['ID', 'Firma', 'Website', platform, platform + '_2', 'alternatives']
    df_se = pd.DataFrame(new_table, columns=header)

    # Create an Excel file
    dt_str_now = datetime.now().strftime("%Y-%m-%d_%H_%M_%S")
    recent_filename = 'Search_Results_' + platform + '_' + dt_str_now + '.xlsx'
    df_se.to_excel(recent_filename)

    driver.quit()