public repository of the unconeginality_preprocessing. Initial publication
This commit is contained in:
Orihara Izaya
@@ -0,0 +1,229 @@
|
||||
"""
|
||||
The `analytic_functions.py` module provides various analytic functions for processing
|
||||
and analyzing comments data. It includes methods to calculate replies, votes, valence,
|
||||
extremity, and Bayesian-corrected measures.
|
||||
|
||||
"""
|
||||
|
||||
from typing import Tuple
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class AnalyticFunctions:
|
||||
"""
|
||||
A class used to encapsulate analytic functions for processing comment data.
|
||||
|
||||
Methods
|
||||
-------
|
||||
calculate_number_of_replies(replies: list) -> int
|
||||
Calculate the number of replies.
|
||||
calculate_totalvotes(upvotes: int, downvotes: int) -> int
|
||||
Calculate the total number of votes.
|
||||
calculate_valence(downvotes: int, totalvotes: int) -> float
|
||||
Calculate the valence of the comments.
|
||||
calculate_extremity(valence: float) -> float
|
||||
Calculate the extremity of the comments.
|
||||
calculate_bayes_correction_general(measure: float, volume: float, weight: float, average_measure: float) -> float
|
||||
Calculate the Bayesian corrected value for a general measure.
|
||||
calculate_bayes_corrected_value_for_quantile_weights(measure: float, totalvotes: int, weights_for_quantiles: dict[str, float]) -> Tuple[float, float, float]
|
||||
Calculate Bayesian corrected values for different quantile weights.
|
||||
get_values_for_quantiles_of_measure(measure_data: pd.Series) -> dict[str, float]
|
||||
Get values for quantiles of a given measure.
|
||||
"""
|
||||
|
||||
def __int__(self):
|
||||
"""
|
||||
Constructs the AnalyticFunctions object.
|
||||
Only passes as the class is just used to encapsulate analytic functions.
|
||||
"""
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def calculate_number_of_replies(replies: list) -> int:
|
||||
"""
|
||||
Calculate the number of replies from a reply list.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
replies : list
|
||||
List containing replies.
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
The number of replies.
|
||||
"""
|
||||
number_of_replies: int = len(replies)
|
||||
return number_of_replies
|
||||
|
||||
@staticmethod
|
||||
def calculate_totalvotes(upvotes: int, downvotes: int) -> int:
|
||||
"""
|
||||
Calculate the total number of votes.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
upvotes : int
|
||||
Number of upvotes.
|
||||
downvotes : int
|
||||
Number of downvotes.
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
The total number of votes.
|
||||
"""
|
||||
totalvotes: int = upvotes + downvotes
|
||||
return totalvotes
|
||||
|
||||
@staticmethod
|
||||
def calculate_valence(downvotes: int, totalvotes: int) -> float:
|
||||
"""
|
||||
Calculate the valence (overall attitude of votes) of a comment.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
downvotes : int
|
||||
Number of downvotes on the comment.
|
||||
totalvotes : int
|
||||
Total number of votes on the comment.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
The valence of the comment.
|
||||
"""
|
||||
if totalvotes == 0:
|
||||
return float("NaN")
|
||||
else:
|
||||
valence: float = -1 * (downvotes / totalvotes - 0.5)
|
||||
return valence
|
||||
|
||||
@staticmethod
|
||||
def calculate_extremity(valence: float) -> float:
|
||||
"""
|
||||
Calculate the extremity of a comment.
|
||||
The extremity is the absolute value of the valence.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
valence : float
|
||||
The valence of the comment.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
The extremity of the comment.
|
||||
"""
|
||||
if valence == float("NaN"):
|
||||
return float("NaN")
|
||||
else:
|
||||
extremity: float = abs(valence)
|
||||
return extremity
|
||||
|
||||
@staticmethod
|
||||
def calculate_bayes_correction_general(
|
||||
measure: float, volume: float, weight: float, average_measure: float
|
||||
) -> float:
|
||||
"""
|
||||
Calculate the Bayesian corrected value for a general measure,
|
||||
correcting for volume of measurements a value is based on.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
measure : float
|
||||
The original measure.
|
||||
volume : float
|
||||
The volume of data points.
|
||||
weight : float
|
||||
The weight to adjust the measure.
|
||||
average_measure : float
|
||||
The average measure used for correction.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
The Bayesian corrected value.
|
||||
"""
|
||||
if volume == 0:
|
||||
return np.nan
|
||||
bayes_corrected_value: float = (
|
||||
volume / (volume + weight) * measure
|
||||
+ weight / (volume + weight) * average_measure
|
||||
)
|
||||
return bayes_corrected_value
|
||||
|
||||
def calculate_bayes_corrected_value_for_quantile_weights(
|
||||
self,
|
||||
measure: float,
|
||||
totalvotes: int,
|
||||
weights_for_quantiles: dict[str, float],
|
||||
) -> Tuple[float, float, float]:
|
||||
"""
|
||||
Calculate Bayesian corrected values for different quantile weights.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
measure : float
|
||||
The original measure.
|
||||
totalvotes : int
|
||||
The total number of votes.
|
||||
weights_for_quantiles : dict[str, float]
|
||||
Dictionary containing weights for different quantiles.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Tuple[float, float, float]
|
||||
Bayesian corrected values for Q25, Q50, and Q75 quantiles.
|
||||
"""
|
||||
weight_q25: float = weights_for_quantiles["weight_q25"]
|
||||
weight_q5: float = weights_for_quantiles["weight_q5"]
|
||||
weight_q75: float = weights_for_quantiles["weight_q75"]
|
||||
|
||||
average_measure: float = weight_q5
|
||||
|
||||
bayes_corrected_q25_measure: float = self.calculate_bayes_correction_general(
|
||||
measure, totalvotes, weight_q25, average_measure
|
||||
)
|
||||
bayes_corrected_q5_measure: float = self.calculate_bayes_correction_general(
|
||||
measure, totalvotes, weight_q5, average_measure
|
||||
)
|
||||
bayes_corrected_q75_measure: float = self.calculate_bayes_correction_general(
|
||||
measure, totalvotes, weight_q75, average_measure
|
||||
)
|
||||
|
||||
return (
|
||||
bayes_corrected_q25_measure,
|
||||
bayes_corrected_q5_measure,
|
||||
bayes_corrected_q75_measure,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_values_for_quantiles_of_measure(
|
||||
measure_data: pd.Series,
|
||||
) -> dict[str, float]:
|
||||
"""
|
||||
Get values for quantiles of a given measure.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
measure_data : pd.Series
|
||||
The measure data as a pandas Series.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, float]
|
||||
Dictionary containing values for Q25, Q50, and Q75 quantiles.
|
||||
"""
|
||||
value_q25: float = measure_data.quantile(q=0.25)
|
||||
value_q5: float = measure_data.mean()
|
||||
value_q75: float = measure_data.quantile(q=0.75)
|
||||
|
||||
results: dict = {
|
||||
"weight_q25": value_q25,
|
||||
"weight_q5": value_q5,
|
||||
"weight_q75": value_q75,
|
||||
}
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
The `article_data_preprocessor.py` module serves as a preprocessing subclass, based on
|
||||
the `DataPreprocessor` abstract base class, for dealing with article data.
|
||||
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
|
||||
from src.data_preprocessor import DataPreprocessor
|
||||
from src.utils import load_json_file
|
||||
|
||||
|
||||
class ArticleDataPreprocessor(DataPreprocessor):
|
||||
"""
|
||||
A class used to preprocess article data.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
Inherits all attributes from the abstract base class DataPreprocessor
|
||||
|
||||
Methods
|
||||
-------
|
||||
preprocess(article_file_path: Path) -> pd.DataFrame
|
||||
Preprocess an article data file and return it as a DataFrame.
|
||||
"""
|
||||
|
||||
def __init__(self, data_folder_article_data: Path):
|
||||
"""
|
||||
Constructs the ArticleDataPreprocessor object.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data_folder_article_data : Path
|
||||
Path object representing the directory where the article data files are located.
|
||||
"""
|
||||
super().__init__(data_folder_article_data, split_word="SPON_article")
|
||||
|
||||
def preprocess(self, article_file_path: Path) -> pd.DataFrame:
|
||||
"""
|
||||
Preprocess an article data file and return it as a DataFrame.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
article_file_path : pathlib.Path
|
||||
The path object representing the article data file to be preprocessed.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
A pandas DataFrame containing the preprocessed article data.
|
||||
"""
|
||||
single_article_dict: dict = load_json_file(article_file_path)
|
||||
return pd.DataFrame([single_article_dict])
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
The `comment_data_preprocessor.py` module provides a subclass of `DataPreprocessor`,
|
||||
specifically for processing comment data files.
|
||||
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
|
||||
from src.data_preprocessor import DataPreprocessor
|
||||
from src.single_comment_file_preprocessor import SingleCommentFilePreprocessor
|
||||
|
||||
|
||||
class CommentsDataPreprocessor(DataPreprocessor):
|
||||
"""
|
||||
A class used to preprocess comment data.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
Inherits all attributes from the abstract base class DataPreprocessor.
|
||||
single_comment_file_preprocessor : SingleCommentFilePreprocessor
|
||||
An instance of the SingleCommentFilePreprocessor class which is used to preprocess individual comment files.
|
||||
|
||||
Methods
|
||||
-------
|
||||
preprocess(comment_file_path: Path) -> pd.DataFrame
|
||||
Preprocess a comment file and return it as a DataFrame.
|
||||
"""
|
||||
|
||||
def __init__(self, data_folder_comment_data: Path):
|
||||
"""
|
||||
Constructs the CommentsDataPreprocessor object.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data_folder_comment_data : Path
|
||||
Path object representing the directory where the comment data files are located.
|
||||
"""
|
||||
super().__init__(data_folder_comment_data, split_word="SPON_comment")
|
||||
self.single_comment_file_preprocessor: SingleCommentFilePreprocessor = (
|
||||
SingleCommentFilePreprocessor()
|
||||
)
|
||||
|
||||
def preprocess(self, comment_file_path: Path) -> pd.DataFrame:
|
||||
"""
|
||||
Preprocess a comment data file and return it as a DataFrame.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comment_file_path : pathlib.Path
|
||||
The path object representing the comment data file to be preprocessed.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
A pandas DataFrame containing the preprocessed comment data.
|
||||
"""
|
||||
return self.single_comment_file_preprocessor.preprocess(comment_file_path)
|
||||
@@ -0,0 +1,321 @@
|
||||
"""
|
||||
The `create_analysis_dataset.py` module merges, processes and saves comment and article data into a final dataset.
|
||||
|
||||
This module provides the AnalysisDatasetCreator class, which performs tasks such as Bayesian correction, merging data,
|
||||
removing irrelevant information, and saving the final analysis dataset.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from src.utils import save_data
|
||||
from src.analytic_functions import AnalyticFunctions
|
||||
|
||||
|
||||
class AnalysisDatasetCreator:
|
||||
"""
|
||||
A class used to create an analysis dataset from comment data and article data.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
comment_data : pd.DataFrame
|
||||
A dataframe containing comment data.
|
||||
article_data : pd.DataFrame
|
||||
A dataframe containing article data.
|
||||
add_extend_article_information : bool
|
||||
Boolean flag to decide whether to add extended information.
|
||||
name_dataset : str
|
||||
The name of the dataset to be created.
|
||||
analytic_functions : AnalyticFunctions
|
||||
An instance of the class AnalyticFunctions.
|
||||
calculate_bayes_corrected_value_for_quantile_weights_vectorized : np.vectorize
|
||||
A numpy vectorized function for calculating Bayesian corrected values.
|
||||
|
||||
Methods
|
||||
-------
|
||||
process_comments_and_articles_into_analysis_dataset()
|
||||
Process comments and articles into a dataset ready for analysis.
|
||||
_merge_comments_and_article_data() -> pd.DataFrame
|
||||
Merge comment and article datasets.
|
||||
_process_comment_data()
|
||||
Process comment data.
|
||||
_process_article_data()
|
||||
Process article data.
|
||||
_calculate_bayesian_corrections()
|
||||
Calculate Bayesian corrections.
|
||||
_calculate_bayesian_corrected_measure(measure_name: str)
|
||||
Calculate Bayesian corrected measures.
|
||||
_get_bayesian_corrected_column_names(measure_name: str)
|
||||
Get the Bayesian-corrected column names.
|
||||
_remove_irrelevant_information_comments()
|
||||
Remove irrelevant information from comments.
|
||||
_set_comment_data_datatypes_for_efficient_storing()
|
||||
Set comment data datatypes for efficient storage.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
comment_data: pd.DataFrame,
|
||||
article_data: pd.DataFrame,
|
||||
add_extend_article_information: bool = True,
|
||||
add_comments_text_body: bool = True,
|
||||
name_dataset: Path = Path(
|
||||
f"{datetime.now().strftime('%Y-%m-%d')}_full_data.parquet"
|
||||
),
|
||||
):
|
||||
"""
|
||||
Constructs the AnalysisDatasetCreator object.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comment_data : pd.DataFrame
|
||||
A dataframe containing the comment data.
|
||||
article_data : pd.DataFrame
|
||||
A dataframe containing the article data.
|
||||
add_extend_article_information : bool
|
||||
Flag to decide whether to add extended information.
|
||||
add_comments_text_body: bool
|
||||
Flag to decide whether to add the text of comments (memory intensive!)
|
||||
name_dataset : str
|
||||
The name of the dataset to be created.
|
||||
"""
|
||||
self.comment_data: pd.DataFrame = comment_data
|
||||
self.article_data: pd.DataFrame = article_data
|
||||
self.add_extend_article_information: bool = add_extend_article_information
|
||||
self.add_comments_text_body: bool = add_comments_text_body
|
||||
self.name_dataset: Path = name_dataset
|
||||
self.analytic_functions: AnalyticFunctions = AnalyticFunctions()
|
||||
self.calculate_bayes_corrected_value_for_quantile_weights_vectorized = np.vectorize(
|
||||
self.analytic_functions.calculate_bayes_corrected_value_for_quantile_weights
|
||||
)
|
||||
|
||||
def process_comments_and_articles_into_analysis_dataset(self):
|
||||
"""
|
||||
Process comment and article data to create a dataset ready for analysis.
|
||||
|
||||
This method sequentially calls other methods to process comment data, process article data,
|
||||
merge them, and finally save the merged dataset.
|
||||
"""
|
||||
self._process_comment_data()
|
||||
self._process_article_data()
|
||||
merged_analysis_dataset: pd.DataFrame = self._merge_comments_and_article_data()
|
||||
save_data(self.name_dataset, merged_analysis_dataset)
|
||||
|
||||
def _merge_comments_and_article_data(self) -> pd.DataFrame:
|
||||
"""
|
||||
Merges the comment and article dataframes on the 'article' field.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
A dataframe containing the merged comment and article data.
|
||||
"""
|
||||
merged_dataset: pd.DataFrame = pd.merge(
|
||||
self.comment_data, self.article_data, on="article", how="left"
|
||||
)
|
||||
|
||||
merged_dataset.rename(
|
||||
columns={
|
||||
"editing.edited": "edited",
|
||||
"channel": "section",
|
||||
"user.id": "user_id",
|
||||
},
|
||||
inplace=True,
|
||||
)
|
||||
|
||||
merged_dataset: pd.DataFrame = self._calculate_derived_time_information(
|
||||
merged_dataset
|
||||
)
|
||||
|
||||
return merged_dataset
|
||||
|
||||
def _process_comment_data(self):
|
||||
"""
|
||||
Perform pre-processing steps to comment data, includes removing irrelevant information
|
||||
and applying Bayesian corrections.
|
||||
"""
|
||||
self._calculate_bayesian_corrections()
|
||||
self.comment_data.rename(
|
||||
columns={
|
||||
"mean_valence_replies": "mean valence of replies",
|
||||
"mean_extremity_replies": "mean extremity of replies",
|
||||
},
|
||||
inplace=True,
|
||||
)
|
||||
self._set_comment_data_datatypes_for_efficient_storing()
|
||||
self._remove_irrelevant_information_comments()
|
||||
|
||||
def _process_article_data(self):
|
||||
"""
|
||||
Perform pre-processing steps on article data, includes removing irrelevant information.
|
||||
"""
|
||||
if self.add_extend_article_information:
|
||||
irrelevant_columns = [
|
||||
"url",
|
||||
"id",
|
||||
"comments_enabled",
|
||||
"date_modified",
|
||||
"date_published",
|
||||
]
|
||||
self.article_data.drop(columns=irrelevant_columns, inplace=True)
|
||||
else:
|
||||
relevant_columns = ["article", "channel", "date_created"]
|
||||
self.article_data = self.article_data[relevant_columns]
|
||||
|
||||
def _calculate_bayesian_corrections(self):
|
||||
"""
|
||||
Apply bayesian corrections to valence, extremity and the mean valence and extremity of replies.
|
||||
"""
|
||||
self._calculate_bayesian_corrected_measure("valence")
|
||||
self._calculate_bayesian_corrected_measure("extremity")
|
||||
self._calculate_bayesian_corrected_measure("mean_valence_replies")
|
||||
self._calculate_bayesian_corrected_measure("mean_extremity_replies")
|
||||
|
||||
def _calculate_bayesian_corrected_measure(self, measure_name: str):
|
||||
"""
|
||||
Calculate the Bayesian corrected measures for the provided measure name.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
measure_name : str
|
||||
The name of the measure (in comment data) to bayesian correct.
|
||||
"""
|
||||
weights = self.analytic_functions.get_values_for_quantiles_of_measure(
|
||||
self.comment_data[measure_name]
|
||||
)
|
||||
|
||||
(
|
||||
col_name_q25,
|
||||
col_name_q50,
|
||||
col_name_q75,
|
||||
) = self._get_bayesian_corrected_column_names(measure_name)
|
||||
|
||||
(
|
||||
self.comment_data[col_name_q25],
|
||||
self.comment_data[col_name_q50],
|
||||
self.comment_data[col_name_q75],
|
||||
) = self.calculate_bayes_corrected_value_for_quantile_weights_vectorized(
|
||||
self.comment_data[measure_name],
|
||||
self.comment_data["totalvotes"],
|
||||
weights,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_bayesian_corrected_column_names(measure_name: str):
|
||||
"""
|
||||
Generate the bayesian corrected column names based on the measure name.
|
||||
|
||||
Parameters:
|
||||
measure_name (str): the name of the measure in self.comment_data dictionary
|
||||
|
||||
Returns:
|
||||
tuple: The corrected column names for q=0.25, q=0.5, and q=0.75
|
||||
"""
|
||||
if measure_name.startswith("mean_") and measure_name.endswith("_replies"):
|
||||
base_name = measure_name.replace("mean_", "").replace("_replies", "")
|
||||
prefix = "mean bayes-corrected"
|
||||
suffix = " of replies"
|
||||
else:
|
||||
base_name = measure_name
|
||||
prefix = "bayes-corrected"
|
||||
suffix = ""
|
||||
|
||||
return (
|
||||
f"{prefix} (q=0.25) {base_name}{suffix}",
|
||||
f"{prefix} (q=0.5) {base_name}{suffix}",
|
||||
f"{prefix} (q=0.75) {base_name}{suffix}",
|
||||
)
|
||||
|
||||
def _remove_irrelevant_information_comments(self):
|
||||
"""
|
||||
Remove data from irrelevant columns in the comment data.
|
||||
"""
|
||||
irrelevant_columns = [
|
||||
"richTextBody",
|
||||
"highlights",
|
||||
"id",
|
||||
"updated_at",
|
||||
"replies",
|
||||
]
|
||||
extended_columns = ["body", "status", "parent_id"]
|
||||
|
||||
self.comment_data.drop(columns=irrelevant_columns, inplace=True)
|
||||
if not self.add_comments_text_body:
|
||||
self.comment_data.drop(columns=extended_columns, inplace=True)
|
||||
|
||||
def _calculate_derived_time_information(
|
||||
self, merged_dataset: pd.DataFrame
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Calculate the daytime of comments as well as the time of commenting after
|
||||
the release of an article.
|
||||
"""
|
||||
|
||||
merged_dataset["created_at"] = pd.to_datetime(merged_dataset["created_at"])
|
||||
merged_dataset["date_created"] = pd.to_datetime(merged_dataset["date_created"])
|
||||
|
||||
merged_dataset["difference"] = (
|
||||
merged_dataset["created_at"] - merged_dataset["date_created"]
|
||||
)
|
||||
merged_dataset["difference"] = merged_dataset["difference"].dt.total_seconds()
|
||||
merged_dataset["comment time after article release (hours)"] = (
|
||||
merged_dataset["difference"] / 3600
|
||||
)
|
||||
merged_dataset["daytime of comment (hours)"] = (
|
||||
merged_dataset["created_at"].dt.hour
|
||||
+ merged_dataset["created_at"].dt.minute / 60
|
||||
)
|
||||
|
||||
merged_dataset.drop(
|
||||
["date_created", "created_at", "difference"], axis=1, inplace=True
|
||||
)
|
||||
|
||||
return merged_dataset
|
||||
|
||||
def _set_comment_data_datatypes_for_efficient_storing(self):
|
||||
"""
|
||||
Change the datatypes of different columns in comment data to more storage efficient types.
|
||||
"""
|
||||
self.comment_data["order"] = self.comment_data["order"].astype("int16")
|
||||
self.comment_data["upvotes"] = self.comment_data["upvotes"].astype("int16")
|
||||
self.comment_data["downvotes"] = self.comment_data["downvotes"].astype("int16")
|
||||
self.comment_data["number O(n+1)-replies"] = self.comment_data[
|
||||
"number O(n+1)-replies"
|
||||
].astype("int16")
|
||||
self.comment_data["totalvotes"] = self.comment_data["totalvotes"].astype(
|
||||
"int16"
|
||||
)
|
||||
self.comment_data["valence"] = self.comment_data["valence"].astype("float32")
|
||||
self.comment_data["bayes-corrected (q=0.25) valence"] = self.comment_data[
|
||||
"bayes-corrected (q=0.25) valence"
|
||||
].astype("float32")
|
||||
self.comment_data["bayes-corrected (q=0.5) valence"] = self.comment_data[
|
||||
"bayes-corrected (q=0.5) valence"
|
||||
].astype("float32")
|
||||
self.comment_data["bayes-corrected (q=0.75) valence"] = self.comment_data[
|
||||
"bayes-corrected (q=0.75) valence"
|
||||
].astype("float32")
|
||||
|
||||
self.comment_data["extremity"] = self.comment_data["extremity"].astype(
|
||||
"float32"
|
||||
)
|
||||
self.comment_data["bayes-corrected (q=0.25) extremity"] = self.comment_data[
|
||||
"bayes-corrected (q=0.25) extremity"
|
||||
].astype("float32")
|
||||
self.comment_data["bayes-corrected (q=0.5) extremity"] = self.comment_data[
|
||||
"bayes-corrected (q=0.5) extremity"
|
||||
].astype("float32")
|
||||
self.comment_data["bayes-corrected (q=0.75) extremity"] = self.comment_data[
|
||||
"bayes-corrected (q=0.75) extremity"
|
||||
].astype("float32")
|
||||
|
||||
self.comment_data["editing.edited"] = (
|
||||
self.comment_data["editing.edited"]
|
||||
.map({"True": True, "False": False})
|
||||
.astype("bool")
|
||||
)
|
||||
self.comment_data["created_at"] = pd.to_datetime(
|
||||
self.comment_data["created_at"]
|
||||
)
|
||||
@@ -0,0 +1,166 @@
|
||||
"""
|
||||
The `data_preprocessor.py` module is an abstract base class for data preprocessing.
|
||||
This class is extended by specific preprocessing classes which implement a `preprocess`
|
||||
method for specific types of data (e.g., articles or comments).
|
||||
|
||||
"""
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
from typing import Optional
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class DataPreprocessor(ABC):
|
||||
"""
|
||||
An abstract base class used as a template for data preprocessing classes.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
data_folder : Path
|
||||
Path object representing the directory where the data files are located.
|
||||
file_format : str
|
||||
The file format extension of the data files (default is ".json").
|
||||
split_word : str
|
||||
The delimiter used to split filenames to extract article IDs.
|
||||
|
||||
Methods
|
||||
-------
|
||||
get_all_data() -> Optional[pd.DataFrame]
|
||||
Collect all data files from a given directory, preprocess them and return
|
||||
a DataFrame containing all processed data.
|
||||
preprocess(file_path: Path) -> pd.DataFrame
|
||||
Abstract method to preprocess a single file.
|
||||
This method should be implemented by specific preprocessing subclasses.
|
||||
_get_list_of_article_ids_from_filenames(all_files) -> list[str]
|
||||
Get a list of article IDs extracted from the filenames.
|
||||
_get_all_files_to_preprocess(data_folder, file_format) -> list[Path]
|
||||
Get all files with given format in data_folder recursively.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, data_folder: Path, file_format: str = ".json", split_word: str = ""
|
||||
):
|
||||
"""
|
||||
Constructs the DataPreprocessor object.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data_folder : Path
|
||||
Path object representing the directory where the data files are located.
|
||||
file_format : str
|
||||
The file format extension of the data files (default is ".json").
|
||||
split_word : str
|
||||
The delimiter used to split filenames to extract article IDs.
|
||||
"""
|
||||
self.data_folder: Path = data_folder
|
||||
self.file_format: str = file_format
|
||||
self.split_word: str = split_word
|
||||
|
||||
def get_all_data(self) -> Optional[pd.DataFrame]:
|
||||
"""
|
||||
Collect all data files from a given directory, preprocess them and return a DataFrame containing all processed data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Optional[pd.DataFrame]
|
||||
A DataFrame containing all processed data, or None if no data files were found.
|
||||
"""
|
||||
list_all_files_to_preprocess: list[Path] = self._get_all_files_to_preprocess(
|
||||
self.data_folder, self.file_format
|
||||
)
|
||||
|
||||
if not list_all_files_to_preprocess:
|
||||
raise ValueError(
|
||||
"no files found please check the directory path given in the settings"
|
||||
)
|
||||
|
||||
list_all_article_ids: list[str] = self._get_list_of_article_ids_from_filenames(
|
||||
list_all_files_to_preprocess
|
||||
)
|
||||
|
||||
with Pool() as pool:
|
||||
list_dataframes: list = []
|
||||
for df in tqdm(
|
||||
pool.imap_unordered(self.preprocess, list_all_files_to_preprocess),
|
||||
total=len(list_all_files_to_preprocess),
|
||||
):
|
||||
if df is not None:
|
||||
list_dataframes.append(df)
|
||||
|
||||
if len(list_dataframes) > 0:
|
||||
data_all: pd.DataFrame = (
|
||||
pd.concat(list_dataframes, keys=list_all_article_ids)
|
||||
.rename_axis(["article", "idx"])
|
||||
.reset_index()
|
||||
)
|
||||
data_all: pd.DataFrame = data_all.drop("idx", axis=1)
|
||||
else:
|
||||
return None
|
||||
|
||||
return data_all
|
||||
|
||||
@abstractmethod
|
||||
def preprocess(self, file_path: Path) -> pd.DataFrame:
|
||||
"""
|
||||
Abstract method to preprocess a single file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : pathlib.Path
|
||||
The path object representing the file to be preprocessed.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
A pandas DataFrame containing the preprocessed data.
|
||||
|
||||
This method should be implemented by specific preprocessing subclasses.
|
||||
"""
|
||||
pass
|
||||
|
||||
def _get_list_of_article_ids_from_filenames(self, all_files) -> list[str]:
|
||||
"""
|
||||
Get a list of article IDs extracted from the filenames.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
all_files : List[Path]
|
||||
List of Path objects representing the files to be preprocessed.
|
||||
|
||||
Returns
|
||||
-------
|
||||
List[str]
|
||||
List of article IDs.
|
||||
"""
|
||||
list_all_article_ids: list = [
|
||||
f.as_posix().replace(self.file_format, "").split(self.split_word, 1)[1]
|
||||
for f in all_files
|
||||
]
|
||||
return list_all_article_ids
|
||||
|
||||
@staticmethod
|
||||
def _get_all_files_to_preprocess(data_folder, file_format) -> list[Path]:
|
||||
"""
|
||||
Get all files with given format in the data_folder path recursively.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data_folder : pathlib.Path
|
||||
The path object to the directory where the data files are located.
|
||||
file_format : Str
|
||||
The file format extension of the data files.
|
||||
|
||||
Returns
|
||||
-------
|
||||
List[Path]
|
||||
List of Path objects representing the files to be preprocessed.
|
||||
"""
|
||||
list_all_files: list = []
|
||||
for root, dirs, files in os.walk(os.path.join("..", data_folder)):
|
||||
for file in files:
|
||||
if file.endswith(file_format):
|
||||
list_all_files.append(Path(root) / file)
|
||||
return list_all_files
|
||||
@@ -0,0 +1,49 @@
|
||||
from pathlib import Path
|
||||
from src.utils import load_json_file
|
||||
|
||||
|
||||
class SettingsLoader:
|
||||
"""
|
||||
A class for handling the loading
|
||||
|
||||
Attributes
|
||||
----------
|
||||
|
||||
Methods
|
||||
-------
|
||||
load_settings_json(filename: str = "analysis_settings.json"):
|
||||
Loads settings from a JSON file.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.settings: dict[str, str] = self.load_settings_json()
|
||||
self.directory_raw_comment_data: Path = Path(
|
||||
self.settings["path_to_comment_data"]
|
||||
)
|
||||
self.directory_raw_article_data: Path = Path(
|
||||
self.settings["path_to_article_data"]
|
||||
)
|
||||
self.add_extend_article_information = self.settings[
|
||||
"flag_add_extend_article_information"
|
||||
]
|
||||
self.add_comment_text_body = self.settings["flag_include_comments_text_body"]
|
||||
|
||||
@staticmethod
|
||||
def load_settings_json(filename: Path = Path("settings.json")) -> dict[str, str]:
|
||||
"""
|
||||
Function to load JSON file containing the settings to run the analysis with.
|
||||
This contains the path to the data.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str
|
||||
The name of the JSON file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, str]
|
||||
Returns the loaded JSON file.
|
||||
"""
|
||||
data: dict[str, str] = load_json_file(filename)
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,366 @@
|
||||
"""
|
||||
The `single_comment_file_preprocessor.py` module provides functionality to preprocess a single comment json file
|
||||
scraped from Spiegel Online. It includes methods for handling missing data, calculating derived
|
||||
measures, un-nesting replies, and cleaning the comment data.
|
||||
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from src.analytic_functions import AnalyticFunctions
|
||||
from src.utils import load_json_file
|
||||
|
||||
|
||||
class SingleCommentFilePreprocessor:
|
||||
"""
|
||||
A class used to preprocess a single comment file from Spiegel Online.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
analytic_functions : AnalyticFunctions
|
||||
An instance of AnalyticFunctions used for various analytic computations.
|
||||
extract_up_and_downvotes_vectorized : np.vectorize
|
||||
Vectorized function to extract upvotes and downvotes.
|
||||
calculate_number_of_replies_vectorized : np.vectorize
|
||||
Vectorized function to calculate the number of replies.
|
||||
calculate_totalvotes_vectorized : np.vectorize
|
||||
Vectorized function to calculate the total number of votes.
|
||||
calculate_valence_vectorized : np.vectorize
|
||||
Vectorized function to calculate the valence of comments.
|
||||
calculate_extremity_vectorized : np.vectorize
|
||||
Vectorized function to calculate the extremity of comments.
|
||||
|
||||
Methods
|
||||
-------
|
||||
preprocess(comment_file_path: Path) -> Optional[pd.DataFrame]
|
||||
Preprocess the comment data file.
|
||||
_get_children_of_comment_entry(comment_entry: dict[str, any]) -> pd.DataFrame
|
||||
Get children of a comment entry.
|
||||
_extract_up_and_downvotes(action_summary: dict) -> Tuple[int, int]
|
||||
Extract upvotes and downvotes from the action summary.
|
||||
_handle_missing_users(comments_data: pd.DataFrame) -> pd.DataFrame
|
||||
Handle missing user data in the comments.
|
||||
_normalize_higher_order(order: int, dataframe_replies: pd.DataFrame, dataframe_ids: pd.DataFrame) -> pd.DataFrame
|
||||
Normalize higher order replies.
|
||||
_unnest_order(comments_data: pd.DataFrame) -> list[pd.DataFrame]
|
||||
Unnest comments by order.
|
||||
_unnest_replies_by_order(comments_data: pd.DataFrame) -> pd.DataFrame
|
||||
Unnest replies by order.
|
||||
_calculate_mean_measure_of_replies(comments_data: pd.DataFrame, column: str) -> pd.DataFrame
|
||||
Calculate the mean of a measure (e.g., valence, extremity) for replies.
|
||||
_calculate_derived_measures(comments_data: pd.DataFrame) -> pd.DataFrame
|
||||
Calculate derived measures like upvotes, downvotes, total votes, valence, extremity, and the number of replies.
|
||||
_clean_comments_data(complete_comments_data: pd.DataFrame) -> pd.DataFrame
|
||||
Clean the comments data by dropping unnecessary columns.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Constructs the SingleCommentFilePreprocessor object.
|
||||
"""
|
||||
self.analytic_functions: AnalyticFunctions = AnalyticFunctions()
|
||||
self.extract_up_and_downvotes_vectorized = np.vectorize(
|
||||
self._extract_up_and_downvotes
|
||||
)
|
||||
self.calculate_number_of_replies_vectorized = np.vectorize(
|
||||
self.analytic_functions.calculate_number_of_replies
|
||||
)
|
||||
self.calculate_totalvotes_vectorized = np.vectorize(
|
||||
self.analytic_functions.calculate_totalvotes
|
||||
)
|
||||
self.calculate_valence_vectorized = np.vectorize(
|
||||
self.analytic_functions.calculate_valence
|
||||
)
|
||||
|
||||
self.calculate_extremity_vectorized = np.vectorize(
|
||||
self.analytic_functions.calculate_extremity
|
||||
)
|
||||
|
||||
def preprocess(self, comment_file_path: Path) -> Optional[pd.DataFrame]:
|
||||
"""
|
||||
Preprocess the comment data file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comment_file_path : Path
|
||||
Path to the JSON file containing the comment data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Optional[pd.DataFrame]
|
||||
Preprocessed comment data if the file contains comment data, otherwise None.
|
||||
"""
|
||||
comment_data_dict: dict = load_json_file(comment_file_path)
|
||||
|
||||
if not comment_data_dict:
|
||||
return None
|
||||
|
||||
comments_data: pd.DataFrame = pd.json_normalize(comment_data_dict)
|
||||
comments_data: pd.DataFrame = self._handle_missing_users(comments_data)
|
||||
comments_data: pd.DataFrame = self._unnest_replies_by_order(comments_data)
|
||||
comments_data: pd.DataFrame = self._calculate_derived_measures(comments_data)
|
||||
cleaned_comments_data: pd.DataFrame = self._clean_comments_data(comments_data)
|
||||
|
||||
return cleaned_comments_data
|
||||
|
||||
@staticmethod
|
||||
def _get_children_of_comment_entry(comment_entry: dict[str, any]) -> pd.DataFrame:
|
||||
"""
|
||||
Get children of a comment entry.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comment_entry : dict[str, any]
|
||||
Dictionary representing a single comment entry.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
DataFrame containing the children of the comment entry.
|
||||
"""
|
||||
data_frame_children: pd.DataFrame = pd.json_normalize(comment_entry)
|
||||
return data_frame_children
|
||||
|
||||
@staticmethod
|
||||
def _extract_up_and_downvotes(action_summary: dict) -> Tuple[int, int]:
|
||||
"""
|
||||
Extract upvotes and downvotes from the action summary.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
action_summary : dict
|
||||
Dictionary containing action summary data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Tuple[int, int]
|
||||
Upvotes and downvotes count.
|
||||
"""
|
||||
upvotes: int = 0
|
||||
downvotes: int = 0
|
||||
if len(action_summary) != 0:
|
||||
for data in action_summary:
|
||||
if data["__typename"] == "UpvoteActionSummary":
|
||||
upvotes: int = data["count"]
|
||||
if data["__typename"] == "DownvoteActionSummary":
|
||||
downvotes: int = data["count"]
|
||||
return upvotes, downvotes
|
||||
|
||||
@staticmethod
|
||||
def _handle_missing_users(comments_data: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Handle missing user data in the comments by creating empty fields to prevent errors.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comments_data : pd.DataFrame
|
||||
DataFrame containing the comments data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
Modified comments data with missing user data handled.
|
||||
"""
|
||||
if "user.username" not in comments_data:
|
||||
comments_data["user.username"]: str = "NaN"
|
||||
comments_data["user.id"]: str = "NaN"
|
||||
comments_data["user.role"]: str = "NaN"
|
||||
|
||||
return comments_data
|
||||
|
||||
def _normalize_higher_order(
|
||||
self, order: int, dataframe_replies: pd.DataFrame, dataframe_ids: pd.DataFrame
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Normalize higher order replies.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
order : int
|
||||
Current order level of replies.
|
||||
dataframe_replies : pd.DataFrame
|
||||
DataFrame containing replies.
|
||||
dataframe_ids : pd.DataFrame
|
||||
DataFrame containing IDs corresponding to higher order replies.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
Normalized higher order replies.
|
||||
"""
|
||||
list_dataframes_higher_order: list[pd.DataFrame] = [
|
||||
self._get_children_of_comment_entry(entry) for entry in dataframe_replies
|
||||
]
|
||||
dataframe_higher_order: pd.DataFrame = (
|
||||
pd.concat(list_dataframes_higher_order, keys=dataframe_ids)
|
||||
.rename_axis(["parent_id", "idx"])
|
||||
.reset_index()
|
||||
)
|
||||
dataframe_higher_order: pd.DataFrame = dataframe_higher_order.drop(
|
||||
"idx", axis=1
|
||||
)
|
||||
dataframe_higher_order["order"]: int = order
|
||||
return dataframe_higher_order
|
||||
|
||||
def _unnest_order(self, comments_data: pd.DataFrame) -> list[pd.DataFrame]:
|
||||
"""
|
||||
Unnest comments by order.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comments_data : pd.DataFrame
|
||||
DataFrame containing comments data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[pd.DataFrame]
|
||||
List of DataFrames with un-nested comments by order.
|
||||
"""
|
||||
list_dataframes_comment_data: list[pd.DataFrame] = [comments_data]
|
||||
for i in range(3):
|
||||
temporary_dataframe = self._normalize_higher_order(
|
||||
i + 1,
|
||||
list_dataframes_comment_data[-1]["replies"],
|
||||
list_dataframes_comment_data[-1]["id"],
|
||||
)
|
||||
if temporary_dataframe.get("replies") is not None:
|
||||
if temporary_dataframe["replies"].any():
|
||||
list_dataframes_comment_data.append(temporary_dataframe)
|
||||
else:
|
||||
break
|
||||
return list_dataframes_comment_data
|
||||
|
||||
def _unnest_replies_by_order(self, comments_data: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Unnest replies by order.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comments_data : pd.DataFrame
|
||||
DataFrame containing comments data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
DataFrame with replies un-nested by order.
|
||||
"""
|
||||
comments_data["order"]: int = 0
|
||||
comments_data["parent_id"]: int = 0
|
||||
list_un_nested_dataframes: list[pd.DataFrame] = self._unnest_order(
|
||||
comments_data
|
||||
)
|
||||
dataframe_full_un_nested_comment_data: pd.DataFrame = (
|
||||
pd.concat(list_un_nested_dataframes).reset_index(drop=True)
|
||||
if list_un_nested_dataframes
|
||||
else comments_data
|
||||
)
|
||||
return dataframe_full_un_nested_comment_data
|
||||
|
||||
@staticmethod
|
||||
def _calculate_mean_measure_of_replies(comments_data: pd.DataFrame, column: str):
|
||||
"""
|
||||
Calculate the mean of a measure (e.g., valence, extremity) for replies.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comments_data : pd.DataFrame
|
||||
DataFrame containing the comments data.
|
||||
column : str
|
||||
The column for which the mean measure of replies is to be calculated.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
DataFrame with the mean measure of replies added.
|
||||
"""
|
||||
data_replies: pd.DataFrame = (
|
||||
comments_data[comments_data["order"] > 0]
|
||||
.groupby("parent_id")
|
||||
.agg({column: "mean"})
|
||||
)
|
||||
|
||||
column_name_mean: str = "mean_" + column + "_replies"
|
||||
|
||||
data_replies.columns = [column_name_mean]
|
||||
data_replies.reset_index(inplace=True)
|
||||
comments_data: pd.DataFrame = comments_data.merge(
|
||||
data_replies, left_on="id", right_on="parent_id", how="left"
|
||||
)
|
||||
|
||||
comments_data.drop("parent_id_y", axis=1, inplace=True)
|
||||
comments_data.rename(columns={"parent_id_x": "parent_id"}, inplace=True)
|
||||
|
||||
return comments_data
|
||||
|
||||
def _calculate_derived_measures(self, comments_data: pd.DataFrame):
|
||||
"""
|
||||
Calculate measures derived from raw data information like totalvotes, valence, extremity,
|
||||
and the number of replies.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
comments_data : pd.DataFrame
|
||||
DataFrame containing the comments data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
DataFrame with derived measures added.
|
||||
"""
|
||||
comments_data["parent_id"]: str = comments_data["parent_id"].astype(str)
|
||||
(
|
||||
comments_data["upvotes"],
|
||||
comments_data["downvotes"],
|
||||
) = self.extract_up_and_downvotes_vectorized(comments_data["action_summaries"])
|
||||
comments_data["totalvotes"]: int = self.calculate_totalvotes_vectorized(
|
||||
comments_data["upvotes"], comments_data["downvotes"]
|
||||
)
|
||||
comments_data["valence"]: float = self.calculate_valence_vectorized(
|
||||
comments_data["downvotes"], comments_data["totalvotes"]
|
||||
)
|
||||
|
||||
comments_data["extremity"]: float = self.calculate_extremity_vectorized(
|
||||
comments_data["valence"]
|
||||
)
|
||||
|
||||
comments_data["valence"]: float = comments_data["valence"].fillna(0)
|
||||
comments_data[
|
||||
"number O(n+1)-replies"
|
||||
]: int = self.calculate_number_of_replies_vectorized(comments_data["replies"])
|
||||
|
||||
comments_data: pd.DataFrame = self._calculate_mean_measure_of_replies(
|
||||
comments_data, "valence"
|
||||
)
|
||||
|
||||
comments_data: pd.DataFrame = self._calculate_mean_measure_of_replies(
|
||||
comments_data, "extremity"
|
||||
)
|
||||
|
||||
return comments_data
|
||||
|
||||
@staticmethod
|
||||
def _clean_comments_data(complete_comments_data):
|
||||
"""
|
||||
Clean the comments data by dropping unnecessary columns.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
complete_comments_data : pd.DataFrame
|
||||
DataFrame containing the complete comments data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
Cleaned DataFrame with unnecessary columns dropped.
|
||||
"""
|
||||
complete_comments_data: pd.DataFrame = complete_comments_data.drop(
|
||||
["action_summaries", "tags", "user.username", "user.role"], axis=1
|
||||
)
|
||||
|
||||
if "user" in complete_comments_data:
|
||||
complete_comments_data = complete_comments_data.drop("user", axis=1)
|
||||
|
||||
return complete_comments_data
|
||||
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
The `utils.py` module provides utility functions for common operations such as loading
|
||||
JSON files and saving data to CSV files.
|
||||
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
|
||||
def load_json_file(file_path: Path) -> dict:
|
||||
"""
|
||||
Load data from a JSON file and return it as a dictionary.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : pathlib.Path
|
||||
Path object representing a JSON file path.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
A dictionary containing the JSON data.
|
||||
"""
|
||||
with file_path.open("r") as file:
|
||||
comment_data: dict = json.loads(file.read())
|
||||
|
||||
return comment_data
|
||||
|
||||
|
||||
def save_data(filepath: Path, data: pd.DataFrame) -> None:
|
||||
"""
|
||||
Save data to a parquet file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filepath : Path
|
||||
The file path (including name) of the parquet file to save the data to.
|
||||
data : pd.DataFrame
|
||||
A pandas DataFrame containing the data to be saved.
|
||||
|
||||
Returns
|
||||
-------
|
||||
None
|
||||
"""
|
||||
data.to_parquet(filepath)
|
||||
Reference in New Issue
Block a user