99 lines
3.9 KiB
YAML
99 lines
3.9 KiB
YAML
# ==============================================================================
|
|
# ArchiveGPT inter-rater reliability -- analysis settings
|
|
# ==============================================================================
|
|
# The only value you normally need to change is paths.data_root.
|
|
#
|
|
# Everything else describes the structure of the ArchiveGPT release
|
|
# (Zenodo record 15640284) and the parameters of the analysis. Override any
|
|
# value at the command line with, e.g.:
|
|
# python run_all.py --data-root /path/to/archive_gpt
|
|
# ==============================================================================
|
|
|
|
paths:
|
|
# ---------------------------------------------------------------------------
|
|
# Root of the extracted ArchiveGPT release. Must contain
|
|
# analysis/data/raw_data/ (the per-session PsychoPy exports)
|
|
# analysis/data/clean_data/ (the published aggregates)
|
|
# Forward slashes work on Windows too. Use "." if this package sits inside
|
|
# the release folder.
|
|
# ---------------------------------------------------------------------------
|
|
data_root: "D:/McDuf/Downloads/archive_gpt"
|
|
|
|
# Locations within data_root. Change only if you reorganised the release.
|
|
raw_data_dir: "analysis/data/raw_data"
|
|
clean_data_dir: "analysis/data/clean_data"
|
|
|
|
# Published clean-data files. These are used ONLY to (a) define which
|
|
# participants were retained in the published sample and (b) validate the
|
|
# reconstruction. The reliability analysis itself reads the raw exports,
|
|
# because the published trial-level file does not carry item identity.
|
|
participants_file: "dprimes_willingness_trust_expertise.csv"
|
|
trial_file: "correctness_per_trial.csv"
|
|
aggregate_file: "accuracy_usefulness.csv"
|
|
|
|
# Results are written here. Relative paths resolve against this config file.
|
|
output_dir: "output"
|
|
|
|
columns:
|
|
# Column names in the raw PsychoPy exports.
|
|
participant: "Participant"
|
|
image: "image"
|
|
desc_type: "desc_type"
|
|
|
|
# The experiment presents 80 descriptions in two blocks of 40. Each block
|
|
# writes to its own slider column; a trial has exactly one of them non-null.
|
|
# List them in block order.
|
|
accuracy: ["slider_q1.response", "slider_q1_2.response"]
|
|
usefulness: ["slider_q2.response", "slider_q2_2.response"]
|
|
|
|
# Column names in the published participants file.
|
|
expert_status: "expert_status"
|
|
expertise_detail: "archive_expertise_status"
|
|
|
|
design:
|
|
# An ITEM is one description: one image shown with one description source.
|
|
# 40 images x 2 sources = 80 items. Presentation order was randomised, so the
|
|
# `trial` column in the published clean data is serial position, NOT item
|
|
# identity. This is why the analysis must start from the raw exports.
|
|
n_images_expected: 40
|
|
n_items_expected: 80
|
|
|
|
# Values of desc_type identifying who wrote the description.
|
|
source_human: "expert" # written by an archivist
|
|
source_model: "ai" # generated by InternVL2
|
|
|
|
# Values of expert_status identifying the two rater panels.
|
|
panel_expert: "expert"
|
|
panel_nonexpert: "non_expert"
|
|
|
|
# Slider bounds, used for reporting only.
|
|
rating_scale: [0, 100]
|
|
|
|
analysis:
|
|
# Which rating to treat as primary. "usefulness" reruns everything on the
|
|
# second slider.
|
|
primary_measure: "accuracy"
|
|
|
|
# Confidence level for all interval estimates.
|
|
alpha: 0.05
|
|
|
|
# Monte Carlo replications for the rater-homogeneity test (Section 5 of the
|
|
# note). 2000 is ample; 200 is enough for a quick look.
|
|
homogeneity_sims: 2000
|
|
|
|
# Seed for every stochastic step, so runs are reproducible.
|
|
seed: 20240607
|
|
|
|
# How many lowest-validity raters to drop in the final rung of the
|
|
# cumulative-adjustment ladder (Section 2 of the note).
|
|
drop_weakest_n: 3
|
|
|
|
validation:
|
|
# Abort if the reconstruction disagrees with the published clean data.
|
|
# Leave this on: it is the check that proves the rebuilt item identity is
|
|
# correct.
|
|
enforce: true
|
|
|
|
# Maximum tolerated absolute difference against the published files.
|
|
tolerance: 1.0e-6
|