157 lines
6.5 KiB
YAML
157 lines
6.5 KiB
YAML
questionnaire: "knowledge"
|
|
label: "Knowledge about AI"
|
|
scales:
|
|
|
|
- name: "subjective_knowledge"
|
|
label: "Subjective knowledge about AI"
|
|
items:
|
|
- id: "subj_know"
|
|
text: "How would you rate your knowledge about AI?"
|
|
inverse: false
|
|
score_range: [1, 5]
|
|
format: "bipolar"
|
|
calculation: "response"
|
|
response_options: "1 = very low, 5 = very high"
|
|
output: "subjective_knowledge"
|
|
reference: "self"
|
|
|
|
- name: "predicted_knowledge"
|
|
label: "Predicted knowledge (self-estimate in percent)"
|
|
items:
|
|
- id: "predict_know_1"
|
|
text: "What percent of these knowledge questions do you expect to answer correctly?"
|
|
inverse: false
|
|
score_range: [0, 100]
|
|
format: "percent"
|
|
calculation: "response"
|
|
response_options: "0-100%"
|
|
output: "predicted_knowledge"
|
|
reference: "self"
|
|
|
|
- name: "objective_knowledge"
|
|
label: "Objective AI knowledge (18 factual items)"
|
|
items:
|
|
- id: "obj_know_1_1"
|
|
text: "LLMs are trained with a large amount of text data (e.g., internet, social media)."
|
|
correct: 1
|
|
- id: "obj_know_1_2"
|
|
text: "LLMs calculate for their answers which word is most likely to come next."
|
|
correct: 1
|
|
- id: "obj_know_1_3"
|
|
text: "The responses of LLMs may be biased (e.g., racially) based on the data they were trained on."
|
|
correct: 1
|
|
- id: "obj_know_1_4"
|
|
text: "The statements of LLMs are always correct."
|
|
correct: 2
|
|
- id: "obj_know_1_5"
|
|
text: "Humans can still easily recognize AI-generated speech as artificial speech."
|
|
correct: 2
|
|
- id: "obj_know_1_6"
|
|
text: "LLMs can intentionally lie and spread false information."
|
|
correct: 2
|
|
- id: "obj_know_1_7"
|
|
text: "Humans can answer questions about a text better than LLMs."
|
|
correct: 2
|
|
- id: "obj_know_1_8"
|
|
text: "LLMs have learned to understand language like a human."
|
|
correct: 2
|
|
- id: "obj_know_1_9"
|
|
text: "LLMs have no real understanding of what they write."
|
|
correct: 1
|
|
- id: "obj_know_1_10"
|
|
text: "In machine learning, two common groups of strategies to train algorithms are supervised and unsupervised learning."
|
|
correct: 1
|
|
- id: "obj_know_1_11"
|
|
text: "Artificial neural networks attempt to fully replicate neural networks in the brain."
|
|
correct: 2
|
|
- id: "obj_know_1_12"
|
|
text: "Using AI, videos can be created that are indistinguishable from videos created by real people."
|
|
correct: 1
|
|
- id: "obj_know_1_13"
|
|
text: "A strong AI can make decisions on its own."
|
|
correct: 1
|
|
- id: "obj_know_1_14"
|
|
text: "Machine learning is based on statistical principles."
|
|
correct: 1
|
|
- id: "obj_know_1_15"
|
|
text: "A chatbot can correctly answer the question 'Will it rain tomorrow?' with a high probability."
|
|
correct: 1
|
|
- id: "obj_know_1_16"
|
|
text: "The language understanding of AI systems does not yet reach that of humans."
|
|
correct: 1
|
|
- id: "obj_know_1_17"
|
|
text: "The automatic generation of texts has already been used for years in journalism and e-commerce, for example."
|
|
correct: 1
|
|
- id: "obj_know_1_18"
|
|
text: "Content created by AI must be legally marked as such."
|
|
correct: 2
|
|
score_range: [1, 2]
|
|
calculation: "sum_correct"
|
|
response_options: "1 = TRUE, 2 = FALSE (participant answer is scored as correct if it matches 'correct')"
|
|
output: "objective_knowledge"
|
|
reference: "Adapted from Said et al., 2022 and Lermann Henestrosa & Kimmerle, 2024"
|
|
retain_single_items: true
|
|
|
|
- name: "objective_knowledge_confidence"
|
|
label: "Confidence in objective knowledge about AI"
|
|
items:
|
|
- id: "obj_know_2_1"
|
|
text: "LLMs are trained with a large amount of text data (e.g., internet, social media)."
|
|
inverse: false
|
|
- id: "obj_know_2_2"
|
|
text: "LLMs calculate for their answers which word is most likely to come next."
|
|
inverse: false
|
|
- id: "obj_know_2_3"
|
|
text: "The responses of LLMs may be biased (e.g., racially) based on the data they were trained on."
|
|
inverse: false
|
|
- id: "obj_know_2_4"
|
|
text: "The statements of LLMs are always correct."
|
|
inverse: false
|
|
- id: "obj_know_2_5"
|
|
text: "Humans can still easily recognize AI-generated speech as artificial speech."
|
|
inverse: false
|
|
- id: "obj_know_2_6"
|
|
text: "LLMs can intentionally lie and spread false information."
|
|
inverse: false
|
|
- id: "obj_know_2_7"
|
|
text: "Humans can answer questions about a text better than LLMs."
|
|
inverse: false
|
|
- id: "obj_know_2_8"
|
|
text: "LLMs have learned to understand language like a human."
|
|
inverse: false
|
|
- id: "obj_know_2_9"
|
|
text: "LLMs have no real understanding of what they write."
|
|
inverse: false
|
|
- id: "obj_know_2_10"
|
|
text: "In machine learning, two common groups of strategies to train algorithms are supervised and unsupervised learning."
|
|
inverse: false
|
|
- id: "obj_know_2_11"
|
|
text: "Artificial neural networks attempt to fully replicate neural networks in the brain."
|
|
inverse: false
|
|
- id: "obj_know_2_12"
|
|
text: "Using AI, videos can be created that are indistinguishable from videos created by real people."
|
|
inverse: false
|
|
- id: "obj_know_2_13"
|
|
text: "A strong AI can make decisions on its own."
|
|
inverse: false
|
|
- id: "obj_know_2_14"
|
|
text: "Machine learning is based on statistical principles."
|
|
inverse: false
|
|
- id: "obj_know_2_15"
|
|
text: "A chatbot can correctly answer the question 'Will it rain tomorrow?' with a high probability."
|
|
inverse: false
|
|
- id: "obj_know_2_16"
|
|
text: "The language understanding of AI systems does not yet reach that of humans."
|
|
inverse: false
|
|
- id: "obj_know_2_17"
|
|
text: "The automatic generation of texts has already been used for years in journalism and e-commerce, for example."
|
|
inverse: false
|
|
- id: "obj_know_2_18"
|
|
text: "Content created by AI must be legally marked as such."
|
|
inverse: false
|
|
score_range: [1, 6]
|
|
format: "Confidence scale"
|
|
calculation: "mean"
|
|
response_options: "1 = I guessed-50%, 2 = 60%, 3 = 70%, 4 = 80%, 5 = 90%, 6 = I am sure-100%"
|
|
output: "objective_knowledge_confidence"
|
|
reference: "Adapted from Said et al., 2022 and Lermann Henestrosa & Kimmerle, 2024" |