Feature request/Bugfix: Added support for two part difference scale which was needed for risk-opportunity scale

Body: - introduced difference scale (and its processing) - changed risk_opportunity_perception.yaml to new format - retained single scale for cognitiv_selfesteem and effects_on_work
2026-07-08 15:26:13 +02:00
parent b53e63f57e
commit 5f04b2165c
6 changed files with 224 additions and 18 deletions
@@ -27,7 +27,7 @@ scales:
    response_options: "1 = strongly disagree, 7 = strongly agree"
    output: "cognitive_selfesteem_thinking"
    reference: "Ward (2013)"
-
+    retain_single_items: true

  - name: "cognitive_selfesteem_memory"
    label: "cognitive_selfesteem memory"
@@ -50,6 +50,7 @@ scales:
    response_options: "1 = strongly disagree, 7 = strongly agree"
    output: "cognitive_selfesteem_memory"
    reference: "Ward (2013)"
+    retain_single_items: true

  - name: "cognitive_selfesteem_transactive_memory"
    label: "cognitive selfesteem transactive memory"
@@ -72,3 +73,4 @@ scales:
    response_options: "1 = strongly disagree, 7 = strongly agree"
    output: "cognitive_selfesteem_transactive_memory"
    reference: "Ward (2013)"
+    retain_single_items: true
@@ -21,3 +21,4 @@ scales:
    response_options: "1 = strongly disagree, 5 = strongly agree"
    output: "effects_on_work"
    reference: "self"
+    retain_single_items: true
@@ -7,30 +7,42 @@ scales:
      - id: "rop_1"
        text: "How concerned, if at all, are you when you think about AI?"
        inverse: false
+        part: "risk"
      - id: "rop_2"
        text: "How pessimistic, if at all, are you when you imagine the future use of AI?"
        inverse: false
+        part: "risk"
      - id: "rop_3"
        text: "How likely would AI negatively influence your life?"
        inverse: false
+        part: "risk"
      - id: "rop_4"
        text: "How likely would you suffer from the implementation of AI into everyday life?"
        inverse: false
+        part: "risk"
      - id: "rop_5"
-        text: " How confident, if at all, do you feel when you think about the potential of AI?"
+        text: "How confident, if at all, do you feel when you think about the potential of AI?"
        inverse: false
+        part: "opportunity"
      - id: "rop_6"
        text: "How optimistic, if at all, are you when you imagine the future use of AI?"
        inverse: false
+        part: "opportunity"
      - id: "rop_7"
        text: "How likely would AI positively influence your life?"
        inverse: false
+        part: "opportunity"
      - id: "rop_8"
        text: "How likely would you benefit from the implementation of AI into everyday life?"
        inverse: false
+        part: "opportunity"
    score_range: [1, 5]
-    format: "bipolar"
-    calculation: "mean"
+    format: "Lickert"
+    calculation: "difference"
+    parts:
+      minuend: "opportunity"    # score = mean(minuend) - mean(subtrahend)
+      subtrahend: "risk"
+    retain_subscales: true       # optional: also emit the two pole means
    response_options: "1 = not at all, 5 = extremely"
-    output: "risk_opportunity_score"
+    output: "risk_opportunity_perception"
    reference: "Adapted from Walpole & Wilson, 2021; Schwesig et al., 2023"
@@ -121,6 +121,17 @@ class DataPreprocessingAllWaves:
                    coalesced=False,
                )

+                for part_name, alpha_value in getattr(
+                    scale_processor, "cronbachs_alpha_by_part", {}
+                ).items():
+                    subscale_column = f"{scale_processor.name}_{part_name}_mean"
+                    self._aggregate_cronbachs_alpha_values(
+                        subscale_column,
+                        alpha_value,
+                        wave_number,
+                        coalesced=False,
+                    )
+
            result_dataframe: pd.DataFrame = pd.concat(
                [data[[participant_id_column]], *scale_dfs], axis=1
            )
@@ -36,6 +36,9 @@ class ScaleProcessor:
        self.logger: Logger = logger
        self.cronbachs_alpha: float | None = None
        self.retain_single_items: bool = scale_config.get("retain_single_items", False)
+        self.parts: dict = scale_config.get("parts", {})
+        self.retain_subscales: bool = scale_config.get("retain_subscales", False)
+        self.cronbachs_alpha_by_part: dict = {}

    def calculate_cronbachs_alpha(
        self, data_frame: pd.DataFrame, item_ids: list
@@ -315,6 +318,12 @@ class ScaleProcessor:

            return result

+        elif self.calculation == "difference":
+            result = self._calculate_difference(data_frame)
+            mask = self.get_subgroup_mask(data_frame)
+            result = result.where(mask, pd.NA)
+            return result
+
        elif self.calculation in ["mean", "sum"]:
            values = data_frame[item_ids].apply(
                lambda col: col.map(self._apply_missing)
@@ -375,9 +384,76 @@ class ScaleProcessor:
            return pd.NA
        return val

+    def _subscale_ids(self, part_name: str) -> list:
+        """Return item ids belonging to a given part, validating presence.
+        Used for difference scales
+
+        Args:
+            part_name (str): Name of the Subscale/Part.
+
+        Returns:
+            list[str]: List of item ids belonging to the part.
+        """
+        ids = [item["id"] for item in self.items if item.get("part") == part_name]
+        if not ids:
+            raise ValueError(f"No items with part '{part_name}' in scale {self.name}")
+        return ids
+
+    def _subscale_mean(self, data_frame: pd.DataFrame, part_name: str) -> pd.Series:
+        """Row-wise mean of one pole, applying missing-value handling.
+        Used for difference scales
+
+        Args:
+            data_frame (pd.DataFrame): the full data dataframe
+            part_name (str): Name of the Subscale/Part.
+
+        Returns:
+            pd.DataFrame: the sub-scaled mean and cronbachs_alpha
+        """
+        ids = self._subscale_ids(part_name)
+        values = data_frame[ids].apply(lambda col: col.map(self._apply_missing))
+        if len(ids) >= 2:
+            self.cronbachs_alpha_by_part[part_name] = self.calculate_cronbachs_alpha(
+                data_frame, ids
+            )
+        return values.mean(axis=1)
+
+    def _calculate_difference(self, data_frame: pd.DataFrame) -> pd.DataFrame:
+        """score = mean(minuend pole) - mean(subtrahend pole).
+        Used for difference scales
+
+        Args:
+            data_frame (pd.DataFrame): the full data dataframe
+
+        Returns:
+            pd.DataFrame: the sub-scaled mean and cronbachs_alpha with the added total score
+        """
+        minuend = self.parts.get("minuend")
+        subtrahend = self.parts.get("subtrahend")
+        if not minuend or not subtrahend:
+            raise ValueError(
+                f"calculation 'difference' requires parts.minuend and "
+                f"parts.subtrahend in scale {self.name}"
+            )
+        minuend_mean = self._subscale_mean(data_frame, minuend)
+        subtrahend_mean = self._subscale_mean(data_frame, subtrahend)
+        result = pd.DataFrame(index=data_frame.index)
+        result[self.output] = minuend_mean - subtrahend_mean
+        if self.retain_subscales:
+            result[f"{self.name}_{minuend}_mean"] = minuend_mean
+            result[f"{self.name}_{subtrahend}_mean"] = subtrahend_mean
+        return result
+

 def _normalize_value(x: Any) -> str | None:
-    """Convert dataframe cell values to normalized string for comparison."""
+    """Convert dataframe cell values to normalized string for comparison.
+
+    Args:
+        x (Any): Value to normalize.
+
+    Returns:
+        str: Normalized string.
+    """
    if pd.isna(x):
        return None
    try:
@@ -42,7 +42,7 @@ def infer_data_type(scale: dict[str, Any], is_composite=False) -> str:
        else:
            return "TEXT"
    calculation = scale.get("calculation", "")
-    if calculation in ("mean", "response", "mapped_mean"):
+    if calculation in ("mean", "response", "mapped_mean", "difference"):
        return "FLOAT"
    if calculation in ("sum", "sum_correct"):
        return "INTEGER"
@@ -92,6 +92,9 @@ def get_aggregate_range(scale: dict[str, Any], is_composite: bool = False) -> st
            max_score = max(mapped_values)
            if item_count > 1:
                return f"{min_score}–{max_score}"
+    if calculation == "difference" and score_range:
+        span = score_range[1] - score_range[0]
+        return f"{-span}–{span}"
    if calculation == "response" and score_range:
        return f"{score_range[0]}–{score_range[1]}"
    if calculation in ("categorical", "ordinal"):
@@ -155,6 +158,44 @@ def get_item_details(scale: dict) -> tuple[list[str], str | None]:

        return lines, None

+    if calculation == "difference":
+        parts = scale.get("parts", {})
+        minuend = parts.get("minuend")
+        subtrahend = parts.get("subtrahend")
+        lines = [
+            f"Score = mean(**{minuend}**) − mean(**{subtrahend}**). "
+            f"Positive = higher {minuend}, negative = higher {subtrahend}.",
+            "",
+        ]
+        grouped: dict = {}
+        for item in items:
+            grouped.setdefault(item.get("part", "unassigned"), []).append(item)
+
+        for part_name in (minuend, subtrahend):
+            lines.append(f"**{part_name} pole:**")
+            for item in grouped.get(part_name, []):
+                text = item.get("text", "")
+                if item.get("inverse", False):
+                    text += " (inversed)"
+                lines.append(f"  - {text}")
+
+        unassigned = [p for p in grouped if p not in (minuend, subtrahend)]
+        for part_name in unassigned:
+            lines.append(f"**⚠ unassigned part '{part_name}':**")
+            for item in grouped[part_name]:
+                lines.append(f"  - {item.get('text', '')}")
+
+        ranges_difference = [
+            item.get("score_range", scale.get("score_range")) for item in items
+        ]
+        score_range_summary = (
+            ranges_difference[0]
+            if ranges_difference
+            and all(r == ranges_difference[0] for r in ranges_difference)
+            else None
+        )
+        return lines, score_range_summary
+
    if calculation in ("categorical", "ordinal") and "response_options" in scale:
        response_options = scale["response_options"]
        missing_response_option = set(scale.get("missing_response_option", []))
@@ -241,6 +282,24 @@ def render_table(table: list, headers: list) -> str:
    return "\n".join(output)


+def format_alpha(alpha_per_wave: Any) -> str:
+    """Format a {wave: alpha} dict as a display string.
+
+    Args:
+        alpha_per_wave (Any): Cronbach alpha values
+
+    Returns:
+        str: Formatted display string for Cronbach alpha
+    """
+    if isinstance(alpha_per_wave, dict) and alpha_per_wave:
+        return ", ".join(
+            f"wave{wave}: {val:.4f}"
+            for wave, val in sorted(alpha_per_wave.items())
+            if val is not None
+        )
+    return "Not applicable"
+
+
 def generate_db_api_reference(
    settings: dict[str, Any],
    logger: Logger,
@@ -298,14 +357,7 @@ def generate_db_api_reference(

                applied_to_group: str = subgroup_scales.get(output, "all")

-                alpha_per_wave = alpha_dict.get(output, {})
-                if isinstance(alpha_per_wave, dict) and alpha_per_wave:
-                    alpha_str = ", ".join(
-                        f"wave{wave}: {val:.4f}"
-                        for wave, val in sorted(alpha_per_wave.items())
-                    )
-                else:
-                    alpha_str = "Not applicable"
+                contributed_columns: list = [output]

                if output not in variables_data:
                    variables_data[output] = {
@@ -320,7 +372,9 @@ def generate_db_api_reference(
                        "Calculation": scale.get("calculation", ""),
                        "Score Range/Categories": range_aggregated_scale,
                        "Description": scale.get("label", ""),
-                        "Cronbach's $\\alpha$": alpha_str,
+                        "Cronbach's $\\alpha$": format_alpha(
+                            alpha_dict.get(output, {})
+                        ),
                        "Applied To Group": applied_to_group,
                        "Reference": reference,
                        "Retain Single Items": (
@@ -335,9 +389,59 @@ def generate_db_api_reference(
                    ):
                        ordinal_categorical_mappings[output] = scale["response_options"]

+                if scale.get("calculation") == "difference" and scale.get(
+                    "retain_subscales", False
+                ):
+                    parts: dict = scale.get("parts", {})
+                    score_range: list | None = scale.get("score_range")
+                    raw_range: str = (
+                        f"{score_range[0]}–{score_range[1]}" if score_range else ""
+                    )
+
+                    for role in ("minuend", "subtrahend"):
+                        part_name = parts.get(role)
+                        if not part_name:
+                            continue
+
+                        sub_col = f"{scale['name']}_{part_name}_mean"
+                        contributed_columns.append(sub_col)
+
+                        if sub_col not in variables_data:
+                            variables_data[sub_col] = {
+                                "Column Name": sub_col,
+                                "Data Type": "FLOAT",
+                                "Waves": [],
+                                "Questionnaire": questionnaire_config.get(
+                                    "questionnaire",
+                                    os.path.basename(questionnaire_file),
+                                ),
+                                "File": os.path.basename(questionnaire_file),
+                                "Scale Name": scale["name"],
+                                "Calculation": f"mean ({part_name} pole)",
+                                "Score Range/Categories": raw_range,
+                                "Description": (
+                                    f"{part_name.capitalize()} subscale mean of "
+                                    f"{scale.get('label', output)}"
+                                ),
+                                "Cronbach's $\\alpha$": format_alpha(
+                                    alpha_dict.get(sub_col, {})
+                                ),
+                                "Applied To Group": applied_to_group,
+                                "Reference": reference,
+                                "Retain Single Items": "No",
+                            }
+                            pole_lines: list = [
+                                item.get("text", "")
+                                + (" (inversed)" if item.get("inverse", False) else "")
+                                for item in scale["items"]
+                                if item.get("part") == part_name
+                            ]
+                            item_details[sub_col] = (pole_lines, score_range)
+
                wave_label = f"wave{wave_number}"
-                if wave_label not in variables_data[output]["Waves"]:
-                    variables_data[output]["Waves"].append(wave_label)
+                for column in contributed_columns:
+                    if wave_label not in variables_data[column]["Waves"]:
+                        variables_data[column]["Waves"].append(wave_label)

        for composite_scale, composite_specifications in wave_config.get(
            "composite_scales", {}