Forecasting Google Scholar statistics for researchers at the Centre for Human Brain Health

import requests
from bs4 import BeautifulSoup
import json

def scrape_names_from_page(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')

    # Find all elements that contain names - this will need to be customized based on the page's HTML structure
    name_elements = soup.find_all('h2')  # Assuming names are within <h2> tags, change as needed

    names = [elem.get_text(strip=True) for elem in name_elements]
    return names

# List of URLs to scrape
urls = [
    "https://www.birmingham.ac.uk/research/centre-for-human-brain-health/chbh-research-themes/lifespan-and-brain-health",
    "https://www.birmingham.ac.uk/research/centre-for-human-brain-health/chbh-research-themes/cognitive-computational-neuroscience",
    "https://www.birmingham.ac.uk/research/centre-for-human-brain-health/chbh-research-themes/social-interaction-and-communication",
    "https://www.birmingham.ac.uk/research/centre-for-human-brain-health/chbh-research-themes/learning-memory-and-performance",
    "https://www.birmingham.ac.uk/research/centre-for-human-brain-health/chbh-research-themes/awareness-consciousness-and-sleep",
    "https://www.birmingham.ac.uk/research/centre-for-human-brain-health/chbh-research-themes/neuroimaging-methods-and-ai"
]

# Initialize an empty set to hold all unique names across themes
unique_names = set()

# Scrape names from each URL
for url in urls:
    names = scrape_names_from_page(url)
    for name in names:
        if 'Dr ' in name or 'Professor ' in name:
            # Clean name and add to the set of unique names
            clean_name = name.replace('Dr ', '').replace('Professor ', '').strip()
            unique_names.add(clean_name)

import serpapi
import json
import pandas as pd

def author_results(author_id):
    print(f"Extracting author results for author ID: {author_id}")

    params = {
        "api_key": "SERP_API_KEY",  # Replace with your actual SerpApi API key
        "engine": "google_scholar_author",  # Author results search engine
        "author_id": author_id,  # Author ID as function input
        "hl": "en"
    }
    
    search = serpapi.search(params) 
    results = search  # Directly use the 'results'

    # Extract the necessary information
    thumbnail = results.get("author", {}).get("thumbnail")
    name = results.get("author", {}).get("name")
    affiliations = results.get("author", {}).get("affiliations")
    email = results.get("author", {}).get("email")
    website = results.get("author", {}).get("website")
    interests = results.get("author", {}).get("interests")
    
    cited_by_table = results.get("cited_by", {}).get("table")
    cited_by_graph = results.get("cited_by", {}).get("graph")
    
    public_access_link = results.get("public_access", {}).get("link")
    available_public_access = results.get("public_access", {}).get("available")
    not_available_public_access = results.get("public_access", {}).get("not_available")
    co_authors = results.get("co_authors")
    
    author_results_data = {
      "thumbnail": thumbnail,
      "name": name,
      "affiliations": affiliations,
      "email": email,
      "website": website,
      "interests": interests,
      "cited_by_table": cited_by_table,
      "cited_by_graph": cited_by_graph,
      "public_access_link": public_access_link,
      "available_public_access": available_public_access,
      "not_available_public_access": not_available_public_access,
      "co_authors": co_authors
    }
    
    # Save the data to a JSON file named after the author_id
    with open(f"{author_id}.json", 'w', encoding='utf-8') as f:
        json.dump(author_results_data, f, ensure_ascii=False, indent=4)
    
    print(f"Data saved to {author_id}.json")
    return author_results_data 

import os
import json
import pandas as pd

# List all JSON files in the current directory
json_files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.json')]

# Initialize a list to store data for each author
authors_data = []

# Process each JSON file
for filename in json_files:
    with open(filename, 'r', encoding='utf-8') as file:
        data = json.load(file)
        
        # Extracting required information
        name = data['name']
        citations_all = next((item for item in data['cited_by_table'] if 'citations' in item), {}).get('citations', {}).get('all', 0)
        h_index_all = next((item for item in data['cited_by_table'] if 'h_index' in item), {}).get('h_index', {}).get('all', 0)
        i10_index_all = next((item for item in data['cited_by_table'] if 'i10_index' in item), {}).get('i10_index', {}).get('all', 0)
        
        # Initialize a dict for the author's citation years data
        citation_years = {'name': name, 'citations': citations_all, 'h-index': h_index_all, 'i10-index': i10_index_all}
        
        # Add citation count per year to the dict
        for item in data['cited_by_graph']:
            year = item['year']
            citations = item['citations']
            citation_years[f'citations_{year}'] = citations
        
        # Append the author's data to the list
        authors_data.append(citation_years)

# Convert the list of data into a DataFrame
df = pd.DataFrame(authors_data)

# Fill NaN values with 0 to indicate years without citations
df.fillna(0, inplace=True)

# Save the DataFrame to a CSV file
df.to_csv('../data/citations_statistics/citation_statistics.csv', index=False, encoding='utf-8')

 python -m ipykernel install --user --name=.venv

python -m notebook

import pandas as pd
from prophet import Prophet

# Load the data from the CSV file
data = pd.read_csv('../data/citation_stats/citation_statistics.csv')

# Remove the 'citations_2024' column
data = data.drop(columns=['citations_2024'])

# Function to perform forecasting for each person
def forecast_citations(row):
    # Extract the relevant citation columns
    citation_cols = [col for col in row.index if col.startswith('citations_')]
    
    # Create a dataframe with the citation data
    citation_data = pd.DataFrame({'ds': citation_cols, 'y': row[citation_cols]})
    
    # Remove rows with zero citations
    citation_data = citation_data[citation_data['y'] != 0]
    
    # Convert the 'ds' column to datetime by extracting the year from the column name
    citation_data['ds'] = pd.to_datetime(citation_data['ds'].str.split('_').str[1], format='%Y')
    
    # Create a Prophet model
    model = Prophet()
    
    # Fit the model to the citation data
    model.fit(citation_data)
    
    # Generate future dates for forecasting
    future_dates = pd.DataFrame({'ds': ['2024-01-01', '2025-01-01', '2026-01-01']})
    
    # Make predictions for the future dates
    forecast = model.predict(future_dates)
    
    # Extract the forecasted values
    predicted_values = forecast['yhat'].values
    
    return pd.Series(predicted_values, index=['prediction_2024', 'prediction_2025', 'prediction_2026'])

# Apply the forecasting function to each row
data[['prediction_2024', 'prediction_2025', 'prediction_2026']] = data.apply(forecast_citations, axis=1)

# Get the citation columns
citation_cols = [col for col in data.columns if col.startswith('citations_')]

# Sort the citation columns in ascending order
citation_cols = sorted(citation_cols, key=lambda x: int(x.split('_')[1]))

# Create a new DataFrame to store the cumulative citations
cumulative_data = pd.DataFrame(index=data.index)

# Calculate the cumulative citations for each year
for i in range(len(citation_cols)):
    if i == 0:
        cumulative_data[f'cumulative_{citation_cols[i].split("_")[1]}'] = data[citation_cols[i]]
    else:
        cumulative_data[f'cumulative_{citation_cols[i].split("_")[1]}'] = cumulative_data[f'cumulative_{citation_cols[i-1].split("_")[1]}'] + data[citation_cols[i]]

# Calculate the cumulative citations for the predicted years
cumulative_data['cumulative_2024'] = cumulative_data[f'cumulative_{citation_cols[-1].split("_")[1]}'] + data['prediction_2024']
cumulative_data['cumulative_2025'] = cumulative_data['cumulative_2024'] + data['prediction_2025']
cumulative_data['cumulative_2026'] = cumulative_data['cumulative_2025'] + data['prediction_2026']

# Concatenate the original data with the cumulative citations
updated_data = pd.concat([data, cumulative_data], axis=1)

# Save the updated data to the same CSV file
updated_data.to_csv('citation_statistics_with_predictions.csv', index=False)