Word-Suggestions: Keytime Subjective Results
library(ARTool)
library(emmeans)
source("./keytime-setup.R")
measured_runs <- read_keytime_runs(measured_only = TRUE)
trusted_participants_runs <- measured_runs %>% filter(is_participant_trusted)1 Attention Check
measured_runs %>% count(suggestions_type, is_participant_trusted)2 Nasa TLX
long_nasa_data <- trusted_participants_runs %>%
gather(
key = "nasa_question",
value = "nasa_value",
factor_key = TRUE,
effort,
frustration,
performance,
mental_demand,
physical_demand,
temporal_demand
) %>%
mutate(
nasa_question = nasa_question %>% as_factor(),
question_label = nasa_question %>%
recode(
mental_demand = "Mental Demand",
physical_demand = "Physical Demand",
temporal_demand = "Temporal Demand",
performance = "Overall Performance",
effort = "Effort",
frustration = "Frustration Level"
)
) %>%
select(
participant,
keytime,
accuracy,
nasa_question,
question_label,
nasa_value,
suggestions_type
)
plot_nasa <- function(filtered_nasa_data) {
nasa_labels = rep('', 20)
nasa_labels[20] = "High"
nasa_labels[1] = "Low"
pd <- position_dodge(0.1)
ggplot(
data = filtered_nasa_data,
aes(
x = accuracy,
y = nasa_value,
color = keytime,
fill = keytime
# group = keytime
)
) +
expand_limits(y = c(5, 100)) +
SCALE_COLOR_KEY_STROKE_DISCRETE +
SCALE_FILL_KEY_STROKE_DISCRETE +
SCALE_X_ACCURACY_DISCRETE +
# custom_line(position=pd) +
# custom_pointrange(position=pd) +
scale_y_continuous("", breaks = seq(5, 100, 5), labels = nasa_labels) +
# guides(color = guide_legend(override.aes = list(linetype = 0))) +
geom_boxplot(outlier.shape = NA) +
# THEME_BOTTOM_LEGEND +
theme(
axis.title.y = element_blank(),
strip.text = element_text(margin = margin(
t = 0,
l = 0,
b = 2,
r = 0
)),
# We need to add a right margin to prevent the legend to stick out, and a negative top
# margin because I have not been able to bring that legend close to the plot in any other way.
legend.margin = margin(
r = 0,
t = -2,
l = 0,
unit = "mm"
)
) +
facet_wrap(vars(question_label)) +
THEME_BOTTOM_LEGEND
}2.1 Inline Suggestions
inline_nasa_plot <- plot_nasa(long_nasa_data %>% filter(suggestions_type == INLINE_SUGGESTION_TYPE))
ggsave(
graph_path("nasa-tlx-inline.pdf"),
plot = inline_nasa_plot,
width = FULL_WIDTH,
height = (FULL_WIDTH - 20) / GOLDEN_RATIO,
units = "mm",
device = cairo_pdf
)
inline_nasa_plot
for(q in levels(long_nasa_data$nasa_question)){
nasa_q_data <- long_nasa_data %>%
filter(nasa_question == q & suggestions_type == INLINE_SUGGESTION_TYPE) %>%
select(participant, nasa_value, accuracy, keytime, nasa_question)
m_nasa = art(nasa_value ~ keytime * accuracy, data=nasa_q_data)
message(paste('\n==============', q, ' (INLINE)=============='))
message('-------- anova --------')
m <- anova(m_nasa, type = 2)
print(m)
anova_tible <- anova_to_tibble(m) %>%
mutate(question = q,
scale = "nasa-tlx",
suggestions = "inline")
if (exists("all_anovas")) {
all_anovas <- union_all(all_anovas, anova_tible)
} else {
all_anovas <- anova_tible
}
message('-------- contrasts key stroke (INLINE) --------')
print(emmeans(artlm(m_nasa, "keytime"), pairwise ~ keytime))
message('-------- contrasts accuracy (INLINE) --------')
print(emmeans(artlm(m_nasa, "accuracy"), pairwise ~ accuracy))
}
============== effort (INLINE)==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 580 0.36459 0.77860
2 accuracy 4 580 0.76329 0.54942
3 keytime:accuracy 12 580 0.41903 0.95630
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 288 14.4 580 260 317
50 302 14.4 580 273 330
100 303 14.3 580 275 331
200 309 14.2 580 281 337
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -13.15 20.4 580 -0.644 0.9178
keytime0 - keytime100 -14.80 20.4 580 -0.727 0.8861
keytime0 - keytime200 -20.55 20.2 580 -1.016 0.7401
keytime50 - keytime100 -1.66 20.4 580 -0.082 0.9998
keytime50 - keytime200 -7.41 20.2 580 -0.366 0.9832
keytime100 - keytime200 -5.75 20.2 580 -0.285 0.9919
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 309 15.9 580 277 340
0.3 311 16.1 580 279 343
0.5 305 16.1 580 273 336
0.7 303 15.8 580 272 334
0.9 276 16.1 580 244 308
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -2.25 22.7 580 -0.099 1.0000
accuracy0.1 - accuracy0.5 4.08 22.6 580 0.180 0.9998
accuracy0.1 - accuracy0.7 6.13 22.4 580 0.274 0.9988
accuracy0.1 - accuracy0.9 32.86 22.7 580 1.448 0.5967
accuracy0.3 - accuracy0.5 6.33 22.8 580 0.278 0.9987
accuracy0.3 - accuracy0.7 8.38 22.6 580 0.371 0.9959
accuracy0.3 - accuracy0.9 35.11 22.8 580 1.537 0.5385
accuracy0.5 - accuracy0.7 2.06 22.5 580 0.091 1.0000
accuracy0.5 - accuracy0.9 28.78 22.8 580 1.263 0.7139
accuracy0.7 - accuracy0.9 26.73 22.6 580 1.184 0.7605
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== frustration (INLINE)==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 580 26.5557 4.1730e-16 ***
2 accuracy 4 580 6.3917 4.8879e-05 ***
3 keytime:accuracy 12 580 1.3334 0.19497
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 209 13.6 580 182 236
50 285 13.6 580 258 312
100 369 13.5 580 343 395
200 336 13.3 580 310 363
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -76.0 19.2 580 -3.957 0.0005
keytime0 - keytime100 -159.9 19.1 580 -8.353 <.0001
keytime0 - keytime200 -127.2 19.0 580 -6.687 <.0001
keytime50 - keytime100 -83.9 19.1 580 -4.382 0.0001
keytime50 - keytime200 -51.2 19.0 580 -2.692 0.0366
keytime100 - keytime200 32.6 19.0 580 1.722 0.3133
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 349 15.7 580 318 380
0.3 310 15.9 580 278 341
0.5 310 15.8 580 278 341
0.7 294 15.5 580 264 325
0.9 239 15.9 580 208 270
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 39.5789 22.3 580 1.775 0.3894
accuracy0.1 - accuracy0.5 39.6777 22.3 580 1.783 0.3844
accuracy0.1 - accuracy0.7 55.0629 22.0 580 2.498 0.0924
accuracy0.1 - accuracy0.9 110.3004 22.3 580 4.946 <.0001
accuracy0.3 - accuracy0.5 0.0988 22.4 580 0.004 1.0000
accuracy0.3 - accuracy0.7 15.4840 22.2 580 0.698 0.9569
accuracy0.3 - accuracy0.9 70.7214 22.4 580 3.151 0.0147
accuracy0.5 - accuracy0.7 15.3852 22.1 580 0.695 0.9575
accuracy0.5 - accuracy0.9 70.6226 22.4 580 3.153 0.0146
accuracy0.7 - accuracy0.9 55.2374 22.2 580 2.490 0.0943
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== performance (INLINE)==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 580 1.9034 0.12786
2 accuracy 4 580 1.4427 0.21842
3 keytime:accuracy 12 580 1.2272 0.26016
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 293 14.4 580 265 321
50 316 14.4 580 287 344
100 317 14.3 580 289 346
200 277 14.1 580 249 304
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -22.76 20.4 580 -1.117 0.6791
keytime0 - keytime100 -24.54 20.3 580 -1.209 0.6214
keytime0 - keytime200 16.38 20.2 580 0.812 0.8490
keytime50 - keytime100 -1.78 20.3 580 -0.088 0.9998
keytime50 - keytime200 39.15 20.2 580 1.939 0.2128
keytime100 - keytime200 40.93 20.1 580 2.035 0.1763
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 306 15.9 580 274 337
0.3 328 16.1 580 296 360
0.5 275 16.1 580 243 306
0.7 300 15.8 580 269 331
0.9 293 16.1 580 262 325
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -22.24 22.7 580 -0.981 0.8640
accuracy0.1 - accuracy0.5 31.03 22.6 580 1.372 0.6461
accuracy0.1 - accuracy0.7 5.46 22.4 580 0.244 0.9992
accuracy0.1 - accuracy0.9 12.30 22.7 580 0.542 0.9828
accuracy0.3 - accuracy0.5 53.26 22.8 580 2.339 0.1340
accuracy0.3 - accuracy0.7 27.70 22.6 580 1.228 0.7350
accuracy0.3 - accuracy0.9 34.53 22.8 580 1.513 0.5542
accuracy0.5 - accuracy0.7 -25.57 22.5 580 -1.136 0.7872
accuracy0.5 - accuracy0.9 -18.73 22.8 580 -0.823 0.9236
accuracy0.7 - accuracy0.9 6.84 22.6 580 0.303 0.9982
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== mental_demand (INLINE)==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 580 4.63251 0.0032729 **
2 accuracy 4 580 0.45274 0.7704255
3 keytime:accuracy 12 580 1.25785 0.2398539
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 257 14.3 580 229 285
50 302 14.3 580 274 330
100 327 14.2 580 299 355
200 316 14.0 580 288 343
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -45.0 20.2 580 -2.222 0.1185
keytime0 - keytime100 -69.8 20.2 580 -3.462 0.0032
keytime0 - keytime200 -58.8 20.0 580 -2.936 0.0181
keytime50 - keytime100 -24.8 20.2 580 -1.232 0.6070
keytime50 - keytime200 -13.9 20.0 580 -0.692 0.9000
keytime100 - keytime200 11.0 20.0 580 0.548 0.9469
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 298 16.0 580 267 330
0.3 291 16.2 580 260 323
0.5 297 16.1 580 265 328
0.7 319 15.8 580 288 350
0.9 296 16.2 580 265 328
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 7.051 22.8 580 0.310 0.9980
accuracy0.1 - accuracy0.5 1.669 22.7 580 0.074 1.0000
accuracy0.1 - accuracy0.7 -20.422 22.5 580 -0.908 0.8938
accuracy0.1 - accuracy0.9 2.024 22.8 580 0.089 1.0000
accuracy0.3 - accuracy0.5 -5.382 22.9 580 -0.236 0.9993
accuracy0.3 - accuracy0.7 -27.473 22.6 580 -1.214 0.7435
accuracy0.3 - accuracy0.9 -5.027 22.9 580 -0.219 0.9995
accuracy0.5 - accuracy0.7 -22.091 22.6 580 -0.978 0.8651
accuracy0.5 - accuracy0.9 0.355 22.9 580 0.016 1.0000
accuracy0.7 - accuracy0.9 22.446 22.6 580 0.992 0.8591
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== physical_demand (INLINE)==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 580 2.8117 0.038752 *
2 accuracy 4 580 2.3457 0.053502 .
3 keytime:accuracy 12 580 1.4972 0.120476
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 275 14.4 580 246 303
50 301 14.4 580 273 329
100 294 14.3 580 266 322
200 332 14.1 580 304 359
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -26.2 20.3 580 -1.290 0.5698
keytime0 - keytime100 -18.9 20.3 580 -0.934 0.7866
keytime0 - keytime200 -57.1 20.1 580 -2.837 0.0243
keytime50 - keytime100 7.3 20.3 580 0.361 0.9840
keytime50 - keytime200 -30.9 20.1 580 -1.535 0.4171
keytime100 - keytime200 -38.2 20.1 580 -1.905 0.2271
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 318 15.9 580 287 350
0.3 286 16.1 580 254 317
0.5 335 16.0 580 303 366
0.7 285 15.7 580 254 315
0.9 279 16.1 580 248 311
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 32.67 22.6 580 1.445 0.5986
accuracy0.1 - accuracy0.5 -16.23 22.6 580 -0.720 0.9519
accuracy0.1 - accuracy0.7 33.81 22.3 580 1.513 0.5541
accuracy0.1 - accuracy0.9 39.13 22.6 580 1.731 0.4157
accuracy0.3 - accuracy0.5 -48.90 22.7 580 -2.154 0.1988
accuracy0.3 - accuracy0.7 1.14 22.5 580 0.051 1.0000
accuracy0.3 - accuracy0.9 6.46 22.7 580 0.284 0.9986
accuracy0.5 - accuracy0.7 50.04 22.4 580 2.231 0.1698
accuracy0.5 - accuracy0.9 55.36 22.7 580 2.439 0.1065
accuracy0.7 - accuracy0.9 5.32 22.5 580 0.237 0.9993
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== temporal_demand (INLINE)==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 580 4.73851 0.0028297 **
2 accuracy 4 580 1.60195 0.1722718
3 keytime:accuracy 12 580 0.49826 0.9161010
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 322 14.3 580 294 350
50 325 14.3 580 297 353
100 298 14.2 580 270 326
200 259 14.0 580 231 286
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -3.66 20.2 580 -0.181 0.9979
keytime0 - keytime100 23.74 20.2 580 1.177 0.6415
keytime0 - keytime200 63.19 20.0 580 3.152 0.0092
keytime50 - keytime100 27.40 20.2 580 1.359 0.5260
keytime50 - keytime200 66.85 20.1 580 3.334 0.0050
keytime100 - keytime200 39.45 20.0 580 1.975 0.1987
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 271 15.9 580 240 303
0.3 301 16.1 580 270 333
0.5 295 16.1 580 264 327
0.7 327 15.8 580 296 358
0.9 307 16.1 580 276 339
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -29.87 22.7 580 -1.318 0.6803
accuracy0.1 - accuracy0.5 -24.08 22.6 580 -1.065 0.8245
accuracy0.1 - accuracy0.7 -55.38 22.4 580 -2.472 0.0984
accuracy0.1 - accuracy0.9 -35.92 22.7 580 -1.584 0.5080
accuracy0.3 - accuracy0.5 5.79 22.8 580 0.254 0.9991
accuracy0.3 - accuracy0.7 -25.51 22.5 580 -1.131 0.7900
accuracy0.3 - accuracy0.9 -6.05 22.8 580 -0.265 0.9989
accuracy0.5 - accuracy0.7 -31.30 22.5 580 -1.391 0.6335
accuracy0.5 - accuracy0.9 -11.84 22.8 580 -0.520 0.9853
accuracy0.7 - accuracy0.9 19.46 22.5 580 0.863 0.9102
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates 2.2 Bar Suggestions
bar_nasa_plot <- plot_nasa(long_nasa_data %>% filter(suggestions_type == BAR_SUGGESTION_TYPE))
ggsave(
graph_path("nasa-tlx-bar.pdf"),
plot = bar_nasa_plot,
width = FULL_WIDTH,
height = (FULL_WIDTH - 20) / GOLDEN_RATIO,
units = "mm",
device = cairo_pdf
)
inline_nasa_plot
for(q in levels(long_nasa_data$nasa_question)){
nasa_q_data <- long_nasa_data %>%
filter(nasa_question == q &
suggestions_type == BAR_SUGGESTION_TYPE) %>%
select(participant, nasa_value, accuracy, keytime, nasa_question)
m_nasa = art(nasa_value ~ keytime * accuracy, data=nasa_q_data)
message(paste('\n==============', q, ' (BAR) =============='))
message('-------- anova --------')
m <- anova(m_nasa, type = 2)
print(m)
anova_tible <- anova_to_tibble(m) %>%
mutate(question = q,
scale = "nasa-tlx",
suggestions = "bar")
if (exists("all_anovas")) {
all_anovas <- union_all(all_anovas, anova_tible)
} else {
all_anovas <- anova_tible
}
message('-------- contrasts key stroke (BAR) --------')
print(emmeans(artlm(m_nasa, "keytime"), pairwise ~ keytime))
message('-------- contrasts accuracy (BAR) --------')
print(emmeans(artlm(m_nasa, "accuracy"), pairwise ~ accuracy))
}
============== effort (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 561 2.0382 0.107453
2 accuracy 4 561 2.1202 0.076969 .
3 keytime:accuracy 12 561 1.3405 0.191196
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 264 14.2 561 236 292
50 288 14.1 561 261 316
100 301 14.0 561 273 328
200 310 13.9 561 283 337
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -24.80 20.0 561 -1.239 0.6023
keytime0 - keytime100 -37.19 19.9 561 -1.865 0.2444
keytime0 - keytime200 -46.33 19.9 561 -2.332 0.0921
keytime50 - keytime100 -12.38 19.9 561 -0.622 0.9252
keytime50 - keytime200 -21.53 19.9 561 -1.085 0.6991
keytime100 - keytime200 -9.15 19.8 561 -0.463 0.9671
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 306 15.6 561 275 336
0.3 317 15.7 561 287 348
0.5 287 15.6 561 256 317
0.7 289 15.9 561 257 320
0.9 257 15.6 561 227 288
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -11.81 22.2 561 -0.533 0.9839
accuracy0.1 - accuracy0.5 18.87 22.1 561 0.855 0.9130
accuracy0.1 - accuracy0.7 17.05 22.3 561 0.764 0.9408
accuracy0.1 - accuracy0.9 48.31 22.1 561 2.184 0.1871
accuracy0.3 - accuracy0.5 30.68 22.1 561 1.387 0.6361
accuracy0.3 - accuracy0.7 28.86 22.4 561 1.290 0.6973
accuracy0.3 - accuracy0.9 60.12 22.2 561 2.713 0.0534
accuracy0.5 - accuracy0.7 -1.82 22.3 561 -0.082 1.0000
accuracy0.5 - accuracy0.9 29.44 22.1 561 1.334 0.6700
accuracy0.7 - accuracy0.9 31.26 22.3 561 1.400 0.6278
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== frustration (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 561 16.88614 1.6319e-10 ***
2 accuracy 4 561 6.34349 5.3656e-05 ***
3 keytime:accuracy 12 561 0.78158 0.66982
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 218 13.7 561 191 244
50 276 13.6 561 249 302
100 333 13.5 561 307 360
200 335 13.4 561 309 362
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -58.12 19.3 561 -3.009 0.0145
keytime0 - keytime100 -115.60 19.2 561 -6.010 <.0001
keytime0 - keytime200 -117.65 19.2 561 -6.138 <.0001
keytime50 - keytime100 -57.48 19.2 561 -2.992 0.0153
keytime50 - keytime200 -59.54 19.1 561 -3.109 0.0106
keytime100 - keytime200 -2.05 19.1 561 -0.108 0.9996
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 319 15.4 561 288 349
0.3 321 15.5 561 291 351
0.5 308 15.4 561 278 338
0.7 279 15.7 561 248 309
0.9 228 15.4 561 198 259
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -2.49 21.9 561 -0.114 1.0000
accuracy0.1 - accuracy0.5 10.22 21.8 561 0.469 0.9901
accuracy0.1 - accuracy0.7 39.93 22.0 561 1.812 0.3676
accuracy0.1 - accuracy0.9 90.30 21.8 561 4.136 0.0004
accuracy0.3 - accuracy0.5 12.71 21.8 561 0.582 0.9777
accuracy0.3 - accuracy0.7 42.42 22.1 561 1.921 0.3073
accuracy0.3 - accuracy0.9 92.79 21.9 561 4.241 0.0003
accuracy0.5 - accuracy0.7 29.71 22.0 561 1.351 0.6591
accuracy0.5 - accuracy0.9 80.08 21.8 561 3.676 0.0024
accuracy0.7 - accuracy0.9 50.37 22.0 561 2.286 0.1510
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== performance (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 561 2.2062 0.086344 .
2 accuracy 4 561 3.1688 0.013660 *
3 keytime:accuracy 12 561 1.9361 0.028008 *
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 267 14.2 561 239 295
50 306 14.2 561 279 334
100 310 14.0 561 283 338
200 280 13.9 561 253 308
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -39.78 20.0 561 -1.984 0.1952
keytime0 - keytime100 -43.64 20.0 561 -2.186 0.1284
keytime0 - keytime200 -13.68 19.9 561 -0.687 0.9019
keytime50 - keytime100 -3.87 19.9 561 -0.194 0.9974
keytime50 - keytime200 26.10 19.9 561 1.313 0.5551
keytime100 - keytime200 29.96 19.8 561 1.514 0.4299
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 275 15.6 561 244 306
0.3 289 15.7 561 258 320
0.5 313 15.5 561 283 344
0.7 324 15.9 561 292 355
0.9 255 15.6 561 224 286
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -14.2 22.1 561 -0.641 0.9683
accuracy0.1 - accuracy0.5 -38.2 22.0 561 -1.734 0.4139
accuracy0.1 - accuracy0.7 -48.7 22.3 561 -2.185 0.1868
accuracy0.1 - accuracy0.9 20.1 22.1 561 0.911 0.8927
accuracy0.3 - accuracy0.5 -24.0 22.1 561 -1.088 0.8127
accuracy0.3 - accuracy0.7 -34.5 22.3 561 -1.546 0.5330
accuracy0.3 - accuracy0.9 34.3 22.1 561 1.550 0.5304
accuracy0.5 - accuracy0.7 -10.5 22.2 561 -0.472 0.9898
accuracy0.5 - accuracy0.9 58.3 22.0 561 2.647 0.0635
accuracy0.7 - accuracy0.9 68.8 22.3 561 3.088 0.0180
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== mental_demand (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 561 8.47427 1.6314e-05 ***
2 accuracy 4 561 1.82722 0.12207
3 keytime:accuracy 12 561 0.84141 0.60754
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 236 14.0 561 208 263
50 287 13.9 561 259 314
100 312 13.8 561 285 339
200 328 13.7 561 301 355
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -51.2 19.7 561 -2.599 0.0472
keytime0 - keytime100 -76.0 19.6 561 -3.873 0.0007
keytime0 - keytime200 -92.4 19.6 561 -4.724 <.0001
keytime50 - keytime100 -24.8 19.6 561 -1.265 0.5857
keytime50 - keytime200 -41.2 19.5 561 -2.107 0.1520
keytime100 - keytime200 -16.4 19.5 561 -0.841 0.8347
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 298 15.7 561 267 329
0.3 290 15.7 561 259 320
0.5 302 15.6 561 272 333
0.7 310 16.0 561 279 342
0.9 256 15.7 561 225 286
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 8.52 22.2 561 0.383 0.9954
accuracy0.1 - accuracy0.5 -4.23 22.1 561 -0.191 0.9997
accuracy0.1 - accuracy0.7 -12.19 22.4 561 -0.545 0.9826
accuracy0.1 - accuracy0.9 42.44 22.2 561 1.914 0.3111
accuracy0.3 - accuracy0.5 -12.75 22.2 561 -0.575 0.9787
accuracy0.3 - accuracy0.7 -20.71 22.4 561 -0.923 0.8878
accuracy0.3 - accuracy0.9 33.92 22.2 561 1.526 0.5457
accuracy0.5 - accuracy0.7 -7.96 22.3 561 -0.357 0.9965
accuracy0.5 - accuracy0.9 46.66 22.1 561 2.109 0.2175
accuracy0.7 - accuracy0.9 54.63 22.4 561 2.441 0.1061
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== physical_demand (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 561 2.12262 0.09629064 .
2 accuracy 4 561 5.34238 0.00031525 ***
3 keytime:accuracy 12 561 0.49309 0.91915669
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 279 14.2 561 251 306
50 293 14.2 561 265 321
100 273 14.0 561 245 301
200 319 13.9 561 291 346
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -14.58 20.0 561 -0.727 0.8861
keytime0 - keytime100 5.52 20.0 561 0.276 0.9926
keytime0 - keytime200 -40.15 19.9 561 -2.018 0.1826
keytime50 - keytime100 20.10 19.9 561 1.008 0.7450
keytime50 - keytime200 -25.57 19.9 561 -1.286 0.5720
keytime100 - keytime200 -45.67 19.8 561 -2.307 0.0976
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 306 15.5 561 275 336
0.3 325 15.6 561 295 356
0.5 303 15.4 561 273 334
0.7 289 15.8 561 258 320
0.9 231 15.5 561 201 262
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -19.64 22.0 561 -0.895 0.8988
accuracy0.1 - accuracy0.5 2.33 21.9 561 0.106 1.0000
accuracy0.1 - accuracy0.7 16.74 22.1 561 0.757 0.9426
accuracy0.1 - accuracy0.9 74.64 21.9 561 3.407 0.0063
accuracy0.3 - accuracy0.5 21.97 21.9 561 1.003 0.8540
accuracy0.3 - accuracy0.7 36.38 22.2 561 1.642 0.4710
accuracy0.3 - accuracy0.9 94.28 22.0 561 4.295 0.0002
accuracy0.5 - accuracy0.7 14.41 22.1 561 0.653 0.9660
accuracy0.5 - accuracy0.9 72.31 21.9 561 3.309 0.0088
accuracy0.7 - accuracy0.9 57.90 22.1 561 2.619 0.0683
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== temporal_demand (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 561 2.36250 0.07034 .
2 accuracy 4 561 0.94143 0.43949
3 keytime:accuracy 12 561 1.17329 0.29885
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 304 14.2 561 277 332
50 312 14.2 561 284 340
100 285 14.0 561 258 313
200 264 13.9 561 236 291
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -7.37 20.0 561 -0.368 0.9830
keytime0 - keytime100 19.30 20.0 561 0.967 0.7682
keytime0 - keytime200 40.74 19.9 561 2.049 0.1715
keytime50 - keytime100 26.67 19.9 561 1.338 0.5392
keytime50 - keytime200 48.11 19.9 561 2.422 0.0742
keytime100 - keytime200 21.44 19.8 561 1.084 0.6995
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 277 15.7 561 246 308
0.3 304 15.8 561 273 335
0.5 297 15.7 561 267 328
0.7 305 16.0 561 273 336
0.9 273 15.7 561 242 304
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -26.964 22.3 561 -1.210 0.7458
accuracy0.1 - accuracy0.5 -20.603 22.2 561 -0.928 0.8859
accuracy0.1 - accuracy0.7 -27.863 22.5 561 -1.241 0.7272
accuracy0.1 - accuracy0.9 4.211 22.2 561 0.189 0.9997
accuracy0.3 - accuracy0.5 6.361 22.2 561 0.286 0.9985
accuracy0.3 - accuracy0.7 -0.899 22.5 561 -0.040 1.0000
accuracy0.3 - accuracy0.9 31.175 22.3 561 1.399 0.6288
accuracy0.5 - accuracy0.7 -7.260 22.4 561 -0.324 0.9976
accuracy0.5 - accuracy0.9 24.814 22.2 561 1.118 0.7970
accuracy0.7 - accuracy0.9 32.074 22.5 561 1.429 0.6094
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates 2.3 Inline vs Bar
for(q in levels(long_nasa_data$nasa_question)) {
nasa_q_data <- long_nasa_data %>%
filter(nasa_question == q) %>%
select(
participant,
nasa_value,
accuracy,
keytime,
nasa_question,
suggestions_type
)
m_nasa = art(nasa_value ~ keytime * accuracy * suggestions_type,
data = nasa_q_data)
message(paste('\n==============', q, '=============='))
message('-------- anova --------')
m <- anova(m_nasa, type = 2)
print(m)
message('-------- contrasts suggestions type --------')
print(emmeans(
artlm(m_nasa, "suggestions_type"),
pairwise ~ suggestions_type
))
}
============== effort ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 1141 2.078446 0.101336
2 accuracy 4 1141 2.878858 0.021781 *
3 suggestions_type 1 1141 0.086964 0.768127
4 keytime:accuracy 12 1141 1.098034 0.357647
5 keytime:suggestions_type 3 1141 0.388899 0.761023
6 accuracy:suggestions_type 4 1141 0.206664 0.934781
7 keytime:accuracy:suggestions_type 12 1141 0.513655 0.907049
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 594 14.1 1141 566 622
BAR 588 14.4 1141 560 616
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR 6.03 20.1 1141 0.300 0.7646
Results are averaged over the levels of: keytime, accuracy
============== frustration ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 1141 42.00655 < 2.22e-16 ***
2 accuracy 4 1141 12.71541 3.9675e-10 ***
3 suggestions_type 1 1141 4.85520 0.027762 *
4 keytime:accuracy 12 1141 1.23627 0.252328
5 keytime:suggestions_type 3 1141 0.64193 0.588118
6 accuracy:suggestions_type 4 1141 0.59081 0.669345
7 keytime:accuracy:suggestions_type 12 1141 0.94798 0.497588
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 569 14.1 1141 541 597
BAR 613 14.4 1141 585 642
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR -44.4 20.1 1141 -2.206 0.0276
Results are averaged over the levels of: keytime, accuracy
============== performance ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 1141 3.90521 0.0086394 **
2 accuracy 4 1141 1.87900 0.1118010
3 suggestions_type 1 1141 6.25991 0.0124892 *
4 keytime:accuracy 12 1141 2.60573 0.0019820 **
5 keytime:suggestions_type 3 1141 0.40825 0.7471056
6 accuracy:suggestions_type 4 1141 2.70606 0.0291183 *
7 keytime:accuracy:suggestions_type 12 1141 0.49677 0.9174976
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 566 14.1 1141 538 594
BAR 617 14.4 1141 588 645
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR -50.4 20.1 1141 -2.504 0.0124
Results are averaged over the levels of: keytime, accuracy
============== mental_demand ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 1141 12.17001 7.6588e-08 ***
2 accuracy 4 1141 1.39140 0.23477
3 suggestions_type 1 1141 0.21177 0.64547
4 keytime:accuracy 12 1141 1.23437 0.25360
5 keytime:suggestions_type 3 1141 0.65515 0.57978
6 accuracy:suggestions_type 4 1141 0.56476 0.68829
7 keytime:accuracy:suggestions_type 12 1141 0.78488 0.66662
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 596 14.2 1141 568 623
BAR 586 14.4 1141 558 614
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR 9.34 20.2 1141 0.462 0.6439
Results are averaged over the levels of: keytime, accuracy
============== physical_demand ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 1141 4.33024 0.0048062 **
2 accuracy 4 1141 5.80728 0.0001257 ***
3 suggestions_type 1 1141 6.10480 0.0136268 *
4 keytime:accuracy 12 1141 0.88018 0.5669906
5 keytime:suggestions_type 3 1141 0.26732 0.8489832
6 accuracy:suggestions_type 4 1141 2.48103 0.0423243 *
7 keytime:accuracy:suggestions_type 12 1141 1.05054 0.3993686
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 567 14.1 1141 539 594
BAR 616 14.4 1141 588 644
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR -49.8 20.1 1141 -2.470 0.0136
Results are averaged over the levels of: keytime, accuracy
============== temporal_demand ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(nasa_value)
Df Df.res F value Pr(>F)
1 keytime 3 1141 6.90260 0.00013172 ***
2 accuracy 4 1141 2.12563 0.07554449 .
3 suggestions_type 1 1141 4.47862 0.03453755 *
4 keytime:accuracy 12 1141 1.09824 0.35747008
5 keytime:suggestions_type 3 1141 0.34315 0.79413168
6 accuracy:suggestions_type 4 1141 0.45278 0.77042240
7 keytime:accuracy:suggestions_type 12 1141 0.57578 0.86294255
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 570 14.1 1141 542 598
BAR 613 14.4 1141 584 641
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR -42.6 20.2 1141 -2.115 0.0347
Results are averaged over the levels of: keytime, accuracy 3 Subjective Questionnaire
3.1 Graph
long_questions_data = trusted_participants_runs %>%
gather(
key = "question",
value = "answer",
factor_key = TRUE,
controls_satisfactory,
suggestions_accuracy,
keyboard_use_efficiency,
suggestion_distraction
) %>%
mutate(
# Make sure answer is a factor.
answer = factor(answer, levels = AGREEMENT_LEVELS, ordered = T),
num_answer = answer %>%
recode(
`Strongly disagree` = -3,
`Disagree` = -2,
`Somewhat disagree` = -1,
`Neither agree nor disagree` = 0,
`Somewhat agree` = 1,
`Agree` = 2,
`Strongly agree` = 3
) %>%
as.numeric(),
question_label = question %>%
recode_factor(
suggestions_accuracy = "Perceived Accuracy",
keyboard_use_efficiency = "Perceived Usability",
controls_satisfactory = "Satisfaction",
suggestion_distraction = "Suggestions' Disruptivity",
.ordered = TRUE
)
) %>%
select(
participant,
question,
question_label,
accuracy,
answer,
num_answer,
keytime,
suggestions_type
)
plot_levels = list(
SCALE_COLOR_KEY_STROKE_DISCRETE,
SCALE_FILL_KEY_STROKE_DISCRETE,
SCALE_X_ACCURACY_DISCRETE,
scale_y_continuous(
limits = c(-3, 3),
breaks = -3:3,
labels = likert_levels <- c("Strongly disagree",
"",
"",
"",
"",
"",
"Strongly agree")
),
geom_boxplot(outlier.shape = NA),
theme(
axis.title.y = element_blank(),
strip.text = element_text(margin = margin(
t = 0,
l = 0,
b = 2,
r = 0
)),
legend.position = "none",
)
)
ggplot(
long_questions_data,
aes(
x = accuracy,
y = num_answer,
color = keytime,
fill = keytime
)
) + plot_levels +
facet_grid(cols = vars(question_label),
rows = vars(suggestions_type))
#
#
# ggsave(
# graph_path("subjective.pdf"),
# width = FULL_WIDTH,
# height = (FULL_WIDTH / 7) * GOLDEN_RATIO,
# units="mm",
# device=cairo_pdf
# )3.2 Stats Inline Suggestions
for(q in levels(long_questions_data$question)){
q_data <- long_questions_data %>%
filter(question == q & suggestions_type == INLINE_SUGGESTION_TYPE)
m_nasa = art(answer ~ keytime * accuracy, data=q_data)
message(paste('\n==============', q, ' (INLINE) =============='))
message('-------- anova --------')
m <- anova(m_nasa, type = 2)
print(m)
all_anovas <- anova_to_tibble(m) %>%
mutate(question = q, scale="questionnaire", suggestions="inline") %>%
union_all(all_anovas)
message('-------- contrasts key stroke delay (INLINE) --------')
print(emmeans(artlm(m_nasa, "keytime"), pairwise ~ keytime))
message('-------- contrasts accuracy (INLINE) --------')
print(emmeans(artlm(m_nasa, "accuracy"), pairwise ~ accuracy))
}
============== controls_satisfactory (INLINE) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 580 13.6694 1.2595e-08 ***
2 accuracy 4 580 21.1730 2.5905e-16 ***
3 keytime:accuracy 12 580 2.1781 0.011514 *
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke delay (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 361 14.0 580 333 388
50 323 14.0 580 296 350
100 245 13.9 580 218 273
200 274 13.7 580 247 301
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 37.9 19.8 580 1.917 0.2220
keytime0 - keytime100 115.5 19.7 580 5.864 <.0001
keytime0 - keytime200 86.8 19.6 580 4.432 0.0001
keytime50 - keytime100 77.6 19.7 580 3.940 0.0005
keytime50 - keytime200 48.9 19.6 580 2.497 0.0615
keytime100 - keytime200 -28.7 19.5 580 -1.472 0.4552
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 210 14.9 580 180 239
0.3 279 15.1 580 250 309
0.5 286 15.1 580 256 316
0.7 331 14.8 580 302 360
0.9 397 15.1 580 367 426
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -69.60 21.3 580 -3.273 0.0099
accuracy0.1 - accuracy0.5 -76.23 21.2 580 -3.593 0.0033
accuracy0.1 - accuracy0.7 -121.27 21.0 580 -5.771 <.0001
accuracy0.1 - accuracy0.9 -186.94 21.3 580 -8.791 <.0001
accuracy0.3 - accuracy0.5 -6.63 21.4 580 -0.310 0.9980
accuracy0.3 - accuracy0.7 -51.67 21.2 580 -2.443 0.1056
accuracy0.3 - accuracy0.9 -117.33 21.4 580 -5.483 <.0001
accuracy0.5 - accuracy0.7 -45.04 21.1 580 -2.134 0.2069
accuracy0.5 - accuracy0.9 -110.71 21.4 580 -5.185 <.0001
accuracy0.7 - accuracy0.9 -65.67 21.2 580 -3.105 0.0170
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== suggestions_accuracy (INLINE) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 580 0.99996 0.3924411
2 accuracy 4 580 64.25597 < 2.22e-16 ***
3 keytime:accuracy 12 580 2.42137 0.0045233 **
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke delay (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 290 14.4 580 262 318
50 312 14.4 580 284 340
100 286 14.3 580 258 314
200 313 14.1 580 285 340
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 -22.335 20.3 580 -1.100 0.6896
keytime0 - keytime100 4.085 20.2 580 0.202 0.9971
keytime0 - keytime200 -22.661 20.1 580 -1.127 0.6731
keytime50 - keytime100 26.419 20.2 580 1.306 0.5595
keytime50 - keytime200 -0.327 20.1 580 -0.016 1.0000
keytime100 - keytime200 -26.746 20.0 580 -1.335 0.5411
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 157 13.3 580 131 183
0.3 240 13.5 580 213 266
0.5 301 13.4 580 275 327
0.7 383 13.2 580 357 408
0.9 421 13.5 580 395 448
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -83.1 18.9 580 -4.386 0.0001
accuracy0.1 - accuracy0.5 -144.3 18.9 580 -7.635 <.0001
accuracy0.1 - accuracy0.7 -225.7 18.7 580 -12.062 <.0001
accuracy0.1 - accuracy0.9 -264.6 18.9 580 -13.972 <.0001
accuracy0.3 - accuracy0.5 -61.2 19.0 580 -3.219 0.0118
accuracy0.3 - accuracy0.7 -142.7 18.8 580 -7.574 <.0001
accuracy0.3 - accuracy0.9 -181.5 19.1 580 -9.525 <.0001
accuracy0.5 - accuracy0.7 -81.5 18.8 580 -4.335 0.0002
accuracy0.5 - accuracy0.9 -120.3 19.0 580 -6.328 <.0001
accuracy0.7 - accuracy0.9 -38.9 18.8 580 -2.063 0.2375
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== keyboard_use_efficiency (INLINE) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 580 56.9430 < 2.22e-16 ***
2 accuracy 4 580 9.8650 9.9936e-08 ***
3 keytime:accuracy 12 580 2.2716 0.0080765 **
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke delay (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 423 12.7 580 398 447
50 332 12.7 580 307 357
100 224 12.6 580 199 249
200 227 12.5 580 202 251
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 90.07 18.0 580 5.014 <.0001
keytime0 - keytime100 198.30 17.9 580 11.080 <.0001
keytime0 - keytime200 195.99 17.8 580 11.015 <.0001
keytime50 - keytime100 108.23 17.9 580 6.046 <.0001
keytime50 - keytime200 105.92 17.8 580 5.952 <.0001
keytime100 - keytime200 -2.31 17.7 580 -0.130 0.9992
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 251 15.5 580 220 281
0.3 282 15.7 580 251 313
0.5 287 15.6 580 256 317
0.7 302 15.3 580 272 332
0.9 382 15.7 580 351 413
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -31.4 22.0 580 -1.427 0.6105
accuracy0.1 - accuracy0.5 -36.0 22.0 580 -1.640 0.4725
accuracy0.1 - accuracy0.7 -51.1 21.8 580 -2.348 0.1314
accuracy0.1 - accuracy0.9 -131.2 22.0 580 -5.958 <.0001
accuracy0.3 - accuracy0.5 -4.6 22.1 580 -0.208 0.9996
accuracy0.3 - accuracy0.7 -19.7 21.9 580 -0.898 0.8975
accuracy0.3 - accuracy0.9 -99.8 22.2 580 -4.502 0.0001
accuracy0.5 - accuracy0.7 -15.1 21.9 580 -0.690 0.9587
accuracy0.5 - accuracy0.9 -95.2 22.1 580 -4.304 0.0002
accuracy0.7 - accuracy0.9 -80.1 21.9 580 -3.657 0.0026
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== suggestion_distraction (INLINE) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 580 5.57349 0.00089617 ***
2 accuracy 4 580 20.30510 1.1414e-15 ***
3 keytime:accuracy 12 580 0.70845 0.74392756
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke delay (INLINE) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 340 14.3 580 312 368
50 315 14.3 580 287 343
100 287 14.2 580 259 314
200 263 14.0 580 235 290
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 24.6 20.2 580 1.220 0.6145
keytime0 - keytime100 52.9 20.1 580 2.630 0.0434
keytime0 - keytime200 76.6 20.0 580 3.832 0.0008
keytime50 - keytime100 28.3 20.1 580 1.406 0.4964
keytime50 - keytime200 52.0 20.0 580 2.600 0.0469
keytime100 - keytime200 23.7 19.9 580 1.191 0.6328
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (INLINE) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 375 15.0 580 345 404
0.3 338 15.2 580 308 368
0.5 325 15.1 580 296 355
0.7 260 14.8 580 231 289
0.9 204 15.2 580 174 234
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 37.0 21.3 580 1.734 0.4141
accuracy0.1 - accuracy0.5 49.7 21.3 580 2.337 0.1349
accuracy0.1 - accuracy0.7 114.9 21.1 580 5.449 <.0001
accuracy0.1 - accuracy0.9 170.8 21.3 580 8.006 <.0001
accuracy0.3 - accuracy0.5 12.8 21.4 580 0.595 0.9758
accuracy0.3 - accuracy0.7 77.9 21.2 580 3.671 0.0024
accuracy0.3 - accuracy0.9 133.8 21.5 580 6.232 <.0001
accuracy0.5 - accuracy0.7 65.2 21.2 580 3.077 0.0185
accuracy0.5 - accuracy0.9 121.1 21.4 580 5.651 <.0001
accuracy0.7 - accuracy0.9 55.9 21.2 580 2.635 0.0655
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates 3.3 Stats Bar Suggestions
for(q in levels(long_questions_data$question)){
q_data <- long_questions_data %>%
filter(question == q &
suggestions_type == BAR_SUGGESTION_TYPE)
m_nasa = art(answer ~ keytime * accuracy, data=q_data)
message(paste('\n==============', q, ' (BAR) =============='))
message('-------- anova --------')
m <- anova(m_nasa, type = 2)
print(m)
all_anovas <- anova_to_tibble(m) %>%
mutate(question = q, scale="questionnaire", suggestions="bar") %>%
union_all(all_anovas)
message('-------- contrasts key stroke (BAR) --------')
print(emmeans(artlm(m_nasa, "keytime"), pairwise ~ keytime))
message('-------- contrasts accuracy (BAR) --------')
print(emmeans(artlm(m_nasa, "accuracy"), pairwise ~ accuracy))
}
============== controls_satisfactory (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 561 4.7168 0.0029225 **
2 accuracy 4 561 16.8668 4.5579e-13 ***
3 keytime:accuracy 12 561 2.3393 0.0062540 **
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 336 14.1 561 308 364
50 284 14.0 561 256 311
100 269 13.9 561 242 297
200 276 13.8 561 249 303
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 52.19 19.9 561 2.628 0.0436
keytime0 - keytime100 66.68 19.8 561 3.372 0.0044
keytime0 - keytime200 60.13 19.7 561 3.051 0.0127
keytime50 - keytime100 14.49 19.8 561 0.734 0.8836
keytime50 - keytime200 7.94 19.7 561 0.403 0.9778
keytime100 - keytime200 -6.55 19.6 561 -0.334 0.9871
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 219 14.9 561 190 248
0.3 278 14.9 561 249 307
0.5 259 14.8 561 230 288
0.7 322 15.2 561 292 351
0.9 379 14.9 561 349 408
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -58.9 21.1 561 -2.794 0.0427
accuracy0.1 - accuracy0.5 -40.2 21.0 561 -1.913 0.3116
accuracy0.1 - accuracy0.7 -102.7 21.2 561 -4.835 <.0001
accuracy0.1 - accuracy0.9 -159.6 21.0 561 -7.584 <.0001
accuracy0.3 - accuracy0.5 18.8 21.0 561 0.891 0.9001
accuracy0.3 - accuracy0.7 -43.8 21.3 561 -2.057 0.2406
accuracy0.3 - accuracy0.9 -100.7 21.1 561 -4.774 <.0001
accuracy0.5 - accuracy0.7 -62.5 21.2 561 -2.950 0.0273
accuracy0.5 - accuracy0.9 -119.5 21.0 561 -5.688 <.0001
accuracy0.7 - accuracy0.9 -56.9 21.2 561 -2.679 0.0584
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== suggestions_accuracy (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 561 0.24308 0.866262
2 accuracy 4 561 53.72285 < 2e-16 ***
3 keytime:accuracy 12 561 2.11316 0.014722 *
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 296 14.2 561 268 324
50 282 14.2 561 255 310
100 297 14.0 561 270 325
200 289 13.9 561 262 316
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 13.63 20.0 561 0.680 0.9046
keytime0 - keytime100 -1.45 20.0 561 -0.073 0.9999
keytime0 - keytime200 7.00 19.9 561 0.352 0.9851
keytime50 - keytime100 -15.09 19.9 561 -0.757 0.8738
keytime50 - keytime200 -6.64 19.9 561 -0.334 0.9871
keytime100 - keytime200 8.45 19.8 561 0.427 0.9739
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 145 13.3 561 119 171
0.3 258 13.4 561 231 284
0.5 297 13.3 561 271 323
0.7 362 13.6 561 336 389
0.9 396 13.3 561 370 422
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -112.6 18.9 561 -5.955 <.0001
accuracy0.1 - accuracy0.5 -151.5 18.8 561 -8.047 <.0001
accuracy0.1 - accuracy0.7 -217.2 19.0 561 -11.405 <.0001
accuracy0.1 - accuracy0.9 -251.1 18.9 561 -13.305 <.0001
accuracy0.3 - accuracy0.5 -38.9 18.9 561 -2.062 0.2383
accuracy0.3 - accuracy0.7 -104.6 19.1 561 -5.482 <.0001
accuracy0.3 - accuracy0.9 -138.5 18.9 561 -7.323 <.0001
accuracy0.5 - accuracy0.7 -65.7 19.0 561 -3.458 0.0053
accuracy0.5 - accuracy0.9 -99.6 18.8 561 -5.288 <.0001
accuracy0.7 - accuracy0.9 -33.9 19.0 561 -1.777 0.3879
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== keyboard_use_efficiency (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 561 26.07936 8.3564e-16 ***
2 accuracy 4 561 1.67581 0.15407
3 keytime:accuracy 12 561 0.91807 0.52832
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 375 13.3 561 349 401
50 315 13.3 561 289 341
100 254 13.2 561 228 280
200 223 13.1 561 197 248
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 60.0 18.8 561 3.185 0.0083
keytime0 - keytime100 121.5 18.8 561 6.475 <.0001
keytime0 - keytime200 152.6 18.7 561 8.161 <.0001
keytime50 - keytime100 61.5 18.7 561 3.280 0.0060
keytime50 - keytime200 92.6 18.7 561 4.957 <.0001
keytime100 - keytime200 31.1 18.6 561 1.673 0.3392
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 268 15.7 561 237 299
0.3 285 15.7 561 254 316
0.5 278 15.6 561 247 309
0.7 309 16.0 561 278 341
0.9 316 15.7 561 285 346
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 -17.28 22.2 561 -0.778 0.9370
accuracy0.1 - accuracy0.5 -9.96 22.1 561 -0.450 0.9915
accuracy0.1 - accuracy0.7 -41.16 22.4 561 -1.839 0.3520
accuracy0.1 - accuracy0.9 -47.53 22.2 561 -2.144 0.2032
accuracy0.3 - accuracy0.5 7.31 22.2 561 0.330 0.9974
accuracy0.3 - accuracy0.7 -23.88 22.4 561 -1.065 0.8244
accuracy0.3 - accuracy0.9 -30.25 22.2 561 -1.361 0.6526
accuracy0.5 - accuracy0.7 -31.20 22.3 561 -1.397 0.6298
accuracy0.5 - accuracy0.9 -37.57 22.1 561 -1.698 0.4359
accuracy0.7 - accuracy0.9 -6.37 22.4 561 -0.285 0.9986
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates
============== suggestion_distraction (BAR) ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 561 2.46852 0.061167 .
2 accuracy 4 561 11.77151 3.4538e-09 ***
3 keytime:accuracy 12 561 0.75305 0.699146
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts key stroke (BAR) --------NOTE: Results may be misleading due to involvement in interactions$emmeans
keytime emmean SE df lower.CL upper.CL
0 318 14.2 561 290 346
50 299 14.1 561 271 327
100 281 14.0 561 254 309
200 267 13.9 561 239 294
Results are averaged over the levels of: accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
keytime0 - keytime50 19.0 20.0 561 0.951 0.7773
keytime0 - keytime100 36.4 19.9 561 1.823 0.2634
keytime0 - keytime200 51.1 19.9 561 2.573 0.0506
keytime50 - keytime100 17.3 19.9 561 0.869 0.8206
keytime50 - keytime200 32.1 19.9 561 1.616 0.3703
keytime100 - keytime200 14.8 19.8 561 0.747 0.8780
Results are averaged over the levels of: accuracy
P value adjustment: tukey method for comparing a family of 4 estimates -------- contrasts accuracy (BAR) --------
NOTE: Results may be misleading due to involvement in interactions$emmeans
accuracy emmean SE df lower.CL upper.CL
0.1 336 15.1 561 307 366
0.3 333 15.2 561 303 362
0.5 305 15.1 561 275 335
0.7 268 15.4 561 238 298
0.9 211 15.1 561 182 241
Results are averaged over the levels of: keytime
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
accuracy0.1 - accuracy0.3 3.73 21.4 561 0.174 0.9998
accuracy0.1 - accuracy0.5 31.36 21.4 561 1.469 0.5834
accuracy0.1 - accuracy0.7 68.16 21.6 561 3.156 0.0145
accuracy0.1 - accuracy0.9 124.86 21.4 561 5.835 <.0001
accuracy0.3 - accuracy0.5 27.63 21.4 561 1.291 0.6967
accuracy0.3 - accuracy0.7 64.44 21.6 561 2.977 0.0252
accuracy0.3 - accuracy0.9 121.14 21.4 561 5.649 <.0001
accuracy0.5 - accuracy0.7 36.80 21.6 561 1.708 0.4300
accuracy0.5 - accuracy0.9 93.50 21.4 561 4.379 0.0001
accuracy0.7 - accuracy0.9 56.70 21.6 561 2.625 0.0672
Results are averaged over the levels of: keytime
P value adjustment: tukey method for comparing a family of 5 estimates 3.4 Inline vs Bar
long_questions_data %>%
filter(question == "suggestion_distraction") %>%
# filter(accuracy == 0.5) %>%
group_by(suggestions_type) %>%
summarise(
n_agree = sum(answer == "Strongly agree" | answer == "Agree"),
total = n(),
ratio = n_agree / total
)for(q in levels(long_questions_data$question)) {
q_data <- long_questions_data %>%
filter(question == q)
m_nasa = art(answer ~ keytime * accuracy * suggestions_type,
data = q_data)
message(paste('\n==============', q, '=============='))
message('-------- anova --------')
m <- anova(m_nasa, type = 2)
print(m)
message('-------- contrasts suggestions type --------')
print(emmeans(
artlm(m_nasa, "suggestions_type"),
pairwise ~ suggestions_type
))
}
============== controls_satisfactory ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 1141 16.86868 1.0023e-10 ***
2 accuracy 4 1141 36.34415 < 2.22e-16 ***
3 suggestions_type 1 1141 3.53184 0.060456 .
4 keytime:accuracy 12 1141 3.42749 5.7535e-05 ***
5 keytime:suggestions_type 3 1141 1.30386 0.271729
6 accuracy:suggestions_type 4 1141 0.42820 0.788349
7 keytime:accuracy:suggestions_type 12 1141 0.86673 0.581003
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 572 14.1 1141 544 600
BAR 610 14.3 1141 582 638
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR -38 20.1 1141 -1.890 0.0590
Results are averaged over the levels of: keytime, accuracy
============== suggestions_accuracy ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 1141 0.31872 0.811856
2 accuracy 4 1141 124.89523 < 2.22e-16 ***
3 suggestions_type 1 1141 3.02236 0.082394 .
4 keytime:accuracy 12 1141 4.02998 3.7134e-06 ***
5 keytime:suggestions_type 3 1141 0.55656 0.643805
6 accuracy:suggestions_type 4 1141 0.88423 0.472649
7 keytime:accuracy:suggestions_type 12 1141 0.33360 0.983189
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 574 14.1 1141 546 601
BAR 609 14.3 1141 581 637
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR -34.9 20.1 1141 -1.743 0.0817
Results are averaged over the levels of: keytime, accuracy
============== keyboard_use_efficiency ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 1141 71.4727 < 2.22e-16 ***
2 accuracy 4 1141 8.5796 8.029e-07 ***
3 suggestions_type 1 1141 0.1699 0.680277
4 keytime:accuracy 12 1141 1.8742 0.033588 *
5 keytime:suggestions_type 3 1141 2.1048 0.097906 .
6 accuracy:suggestions_type 4 1141 1.3499 0.249409
7 keytime:accuracy:suggestions_type 12 1141 1.3011 0.211472
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 587 14.1 1141 559 614
BAR 595 14.4 1141 567 623
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR -8.38 20.2 1141 -0.416 0.6776
Results are averaged over the levels of: keytime, accuracy
============== suggestion_distraction ==============-------- anova --------Analysis of Variance of Aligned Rank Transformed Data
Table Type: Anova Table (Type II tests)
Model: No Repeated Measures (lm)
Response: art(answer)
Df Df.res F value Pr(>F)
1 keytime 3 1141 7.13381 9.5104e-05 ***
2 accuracy 4 1141 29.78317 < 2.22e-16 ***
3 suggestions_type 1 1141 8.56853 0.0034881 **
4 keytime:accuracy 12 1141 0.77512 0.6767465
5 keytime:suggestions_type 3 1141 0.29949 0.8257887
6 accuracy:suggestions_type 4 1141 0.58150 0.6761007
7 keytime:accuracy:suggestions_type 12 1141 0.53772 0.8909947
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 -------- contrasts suggestions type --------NOTE: Results may be misleading due to involvement in interactions$emmeans
suggestions_type emmean SE df lower.CL upper.CL
INLINE 620 14.1 1141 592 648
BAR 561 14.3 1141 533 589
Results are averaged over the levels of: keytime, accuracy
Confidence level used: 0.95
$contrasts
contrast estimate SE df t.ratio p.value
INLINE - BAR 59 20.1 1141 2.932 0.0034
Results are averaged over the levels of: keytime, accuracy 4 Export
LS0tCnRpdGxlOiAiV29yZC1TdWdnZXN0aW9uczogS2V5dGltZSBTdWJqZWN0aXZlIFJlc3VsdHMiCm91dHB1dDoKICBodG1sX25vdGVib29rOgogICAgdG9jOiB0cnVlCiAgICB0b2NfZmxvYXQ6IHRydWUKICAgIHRoZW1lOiBsdW1lbgogICAgaGlnaGxpZ2h0OiBkZWZhdWx0CiAgICBjb2RlX2ZvbGRpbmc6IGhpZGUKICAgIG51bWJlcl9zZWN0aW9uczogVFJVRQotLS0KCgpbR28gYmFjayB0byBtYWluXSguL21haW4ubmIuaHRtbCkKCgpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQpsaWJyYXJ5KEFSVG9vbCkKbGlicmFyeShlbW1lYW5zKQpzb3VyY2UoIi4va2V5dGltZS1zZXR1cC5SIikKCm1lYXN1cmVkX3J1bnMgPC0gcmVhZF9rZXl0aW1lX3J1bnMobWVhc3VyZWRfb25seSA9IFRSVUUpCnRydXN0ZWRfcGFydGljaXBhbnRzX3J1bnMgPC0gbWVhc3VyZWRfcnVucyAlPiUgZmlsdGVyKGlzX3BhcnRpY2lwYW50X3RydXN0ZWQpCmBgYAoKIyBBdHRlbnRpb24gQ2hlY2sKCmBgYHtyfQptZWFzdXJlZF9ydW5zICU+JSBjb3VudChzdWdnZXN0aW9uc190eXBlLCBpc19wYXJ0aWNpcGFudF90cnVzdGVkKQpgYGAKCgojIE5hc2EgVExYCgpgYGB7ciwgZGV2PSJzdmdsaXRlIn0KbG9uZ19uYXNhX2RhdGEgPC0gdHJ1c3RlZF9wYXJ0aWNpcGFudHNfcnVucyAlPiUKICBnYXRoZXIoCiAgICBrZXkgPSAibmFzYV9xdWVzdGlvbiIsCiAgICB2YWx1ZSA9ICJuYXNhX3ZhbHVlIiwKICAgIGZhY3Rvcl9rZXkgPSBUUlVFLAogICAgZWZmb3J0LAogICAgZnJ1c3RyYXRpb24sCiAgICBwZXJmb3JtYW5jZSwKICAgIG1lbnRhbF9kZW1hbmQsCiAgICBwaHlzaWNhbF9kZW1hbmQsCiAgICB0ZW1wb3JhbF9kZW1hbmQKICApICU+JQogIG11dGF0ZSgKICAgIG5hc2FfcXVlc3Rpb24gPSBuYXNhX3F1ZXN0aW9uICU+JSBhc19mYWN0b3IoKSwKICAgIHF1ZXN0aW9uX2xhYmVsID0gbmFzYV9xdWVzdGlvbiAlPiUKICAgICAgcmVjb2RlKAogICAgICAgIG1lbnRhbF9kZW1hbmQgPSAiTWVudGFsIERlbWFuZCIsCiAgICAgICAgcGh5c2ljYWxfZGVtYW5kID0gIlBoeXNpY2FsIERlbWFuZCIsCiAgICAgICAgdGVtcG9yYWxfZGVtYW5kID0gIlRlbXBvcmFsIERlbWFuZCIsCiAgICAgICAgcGVyZm9ybWFuY2UgPSAiT3ZlcmFsbCBQZXJmb3JtYW5jZSIsCiAgICAgICAgZWZmb3J0ID0gIkVmZm9ydCIsCiAgICAgICAgZnJ1c3RyYXRpb24gPSAiRnJ1c3RyYXRpb24gTGV2ZWwiCiAgICAgICkKICApICU+JQogIHNlbGVjdCgKICAgIHBhcnRpY2lwYW50LAogICAga2V5dGltZSwKICAgIGFjY3VyYWN5LAogICAgbmFzYV9xdWVzdGlvbiwKICAgIHF1ZXN0aW9uX2xhYmVsLAogICAgbmFzYV92YWx1ZSwKICAgIHN1Z2dlc3Rpb25zX3R5cGUKICApCgpwbG90X25hc2EgPC0gZnVuY3Rpb24oZmlsdGVyZWRfbmFzYV9kYXRhKSB7CiAgbmFzYV9sYWJlbHMgPSByZXAoJycsIDIwKQogIG5hc2FfbGFiZWxzWzIwXSA9ICJIaWdoIgogIG5hc2FfbGFiZWxzWzFdID0gIkxvdyIKICAKICBwZCA8LSBwb3NpdGlvbl9kb2RnZSgwLjEpCiAgZ2dwbG90KAogICAgZGF0YSA9IGZpbHRlcmVkX25hc2FfZGF0YSwKICAgIGFlcygKICAgICAgeCA9IGFjY3VyYWN5LAogICAgICB5ID0gbmFzYV92YWx1ZSwKICAgICAgY29sb3IgPSBrZXl0aW1lLAogICAgICBmaWxsID0ga2V5dGltZQogICAgICAjIGdyb3VwID0ga2V5dGltZQogICAgKQogICkgKwogICAgZXhwYW5kX2xpbWl0cyh5ID0gYyg1LCAxMDApKSArCiAgICBTQ0FMRV9DT0xPUl9LRVlfU1RST0tFX0RJU0NSRVRFICsKICAgIFNDQUxFX0ZJTExfS0VZX1NUUk9LRV9ESVNDUkVURSArCiAgICBTQ0FMRV9YX0FDQ1VSQUNZX0RJU0NSRVRFICsKICAgICMgY3VzdG9tX2xpbmUocG9zaXRpb249cGQpICsKICAgICMgY3VzdG9tX3BvaW50cmFuZ2UocG9zaXRpb249cGQpICsKICAgIHNjYWxlX3lfY29udGludW91cygiIiwgYnJlYWtzID0gc2VxKDUsIDEwMCwgNSksIGxhYmVscyA9IG5hc2FfbGFiZWxzKSArCiAgICAjIGd1aWRlcyhjb2xvciA9IGd1aWRlX2xlZ2VuZChvdmVycmlkZS5hZXMgPSBsaXN0KGxpbmV0eXBlID0gMCkpKSArCiAgICBnZW9tX2JveHBsb3Qob3V0bGllci5zaGFwZSA9IE5BKSArCiAgICAjIFRIRU1FX0JPVFRPTV9MRUdFTkQgKwogICAgdGhlbWUoCiAgICAgIGF4aXMudGl0bGUueSA9IGVsZW1lbnRfYmxhbmsoKSwKICAgICAgc3RyaXAudGV4dCA9IGVsZW1lbnRfdGV4dChtYXJnaW4gPSBtYXJnaW4oCiAgICAgICAgdCA9IDAsCiAgICAgICAgbCA9IDAsCiAgICAgICAgYiA9IDIsCiAgICAgICAgciA9IDAKICAgICAgKSksCiAgICAgICMgV2UgbmVlZCB0byBhZGQgYSByaWdodCBtYXJnaW4gdG8gcHJldmVudCB0aGUgbGVnZW5kIHRvIHN0aWNrIG91dCwgYW5kIGEgbmVnYXRpdmUgdG9wCiAgICAgICMgbWFyZ2luIGJlY2F1c2UgSSBoYXZlIG5vdCBiZWVuIGFibGUgdG8gYnJpbmcgdGhhdCBsZWdlbmQgY2xvc2UgdG8gdGhlIHBsb3QgaW4gYW55IG90aGVyIHdheS4KICAgICAgbGVnZW5kLm1hcmdpbiA9IG1hcmdpbigKICAgICAgICByID0gMCwKICAgICAgICB0ID0gLTIsCiAgICAgICAgbCA9IDAsCiAgICAgICAgdW5pdCA9ICJtbSIKICAgICAgKQogICAgKSArCiAgICBmYWNldF93cmFwKHZhcnMocXVlc3Rpb25fbGFiZWwpKSArCiAgICBUSEVNRV9CT1RUT01fTEVHRU5ECn0KYGBgCgoKIyMgSW5saW5lIFN1Z2dlc3Rpb25zCgpgYGB7ciwgZGV2PSJzdmdsaXRlIn0KaW5saW5lX25hc2FfcGxvdCA8LSBwbG90X25hc2EobG9uZ19uYXNhX2RhdGEgJT4lIGZpbHRlcihzdWdnZXN0aW9uc190eXBlID09IElOTElORV9TVUdHRVNUSU9OX1RZUEUpKQoKZ2dzYXZlKAogIGdyYXBoX3BhdGgoIm5hc2EtdGx4LWlubGluZS5wZGYiKSwKICBwbG90ID0gaW5saW5lX25hc2FfcGxvdCwKICB3aWR0aCA9IEZVTExfV0lEVEgsCiAgaGVpZ2h0ID0gKEZVTExfV0lEVEggLSAyMCkgLyBHT0xERU5fUkFUSU8sCiAgdW5pdHMgPSAibW0iLAogIGRldmljZSA9IGNhaXJvX3BkZgopCgppbmxpbmVfbmFzYV9wbG90CmBgYAoKYGBge3IsIGRldj0ic3ZnbGl0ZSJ9CmZvcihxIGluIGxldmVscyhsb25nX25hc2FfZGF0YSRuYXNhX3F1ZXN0aW9uKSl7CiAgbmFzYV9xX2RhdGEgPC0gbG9uZ19uYXNhX2RhdGEgJT4lCiAgICBmaWx0ZXIobmFzYV9xdWVzdGlvbiA9PSBxICYgc3VnZ2VzdGlvbnNfdHlwZSA9PSBJTkxJTkVfU1VHR0VTVElPTl9UWVBFKSAlPiUgCiAgICBzZWxlY3QocGFydGljaXBhbnQsIG5hc2FfdmFsdWUsIGFjY3VyYWN5LCBrZXl0aW1lLCBuYXNhX3F1ZXN0aW9uKQogIG1fbmFzYSA9IGFydChuYXNhX3ZhbHVlIH4ga2V5dGltZSAqIGFjY3VyYWN5LCBkYXRhPW5hc2FfcV9kYXRhKQoKICBtZXNzYWdlKHBhc3RlKCdcbj09PT09PT09PT09PT09JywgcSwgJyAoSU5MSU5FKT09PT09PT09PT09PT09JykpCiAgCiAgbWVzc2FnZSgnLS0tLS0tLS0gYW5vdmEgLS0tLS0tLS0nKQogIG0gPC0gYW5vdmEobV9uYXNhLCB0eXBlID0gMikKICBwcmludChtKQogIGFub3ZhX3RpYmxlIDwtIGFub3ZhX3RvX3RpYmJsZShtKSAlPiUKICAgIG11dGF0ZShxdWVzdGlvbiA9IHEsCiAgICAgICAgICAgc2NhbGUgPSAibmFzYS10bHgiLAogICAgICAgICAgIHN1Z2dlc3Rpb25zID0gImlubGluZSIpCiAgCiAgaWYgKGV4aXN0cygiYWxsX2Fub3ZhcyIpKSB7CiAgICBhbGxfYW5vdmFzIDwtIHVuaW9uX2FsbChhbGxfYW5vdmFzLCBhbm92YV90aWJsZSkKICB9IGVsc2UgewogICAgYWxsX2Fub3ZhcyA8LSBhbm92YV90aWJsZQogIH0KICAKICBtZXNzYWdlKCctLS0tLS0tLSBjb250cmFzdHMga2V5IHN0cm9rZSAoSU5MSU5FKSAtLS0tLS0tLScpCiAgcHJpbnQoZW1tZWFucyhhcnRsbShtX25hc2EsICJrZXl0aW1lIiksIHBhaXJ3aXNlIH4ga2V5dGltZSkpCgogIG1lc3NhZ2UoJy0tLS0tLS0tIGNvbnRyYXN0cyBhY2N1cmFjeSAoSU5MSU5FKSAtLS0tLS0tLScpCiAgcHJpbnQoZW1tZWFucyhhcnRsbShtX25hc2EsICJhY2N1cmFjeSIpLCBwYWlyd2lzZSB+IGFjY3VyYWN5KSkKfQpgYGAKCgojIyBCYXIgU3VnZ2VzdGlvbnMKCmBgYHtyLCBkZXY9InN2Z2xpdGUifQpiYXJfbmFzYV9wbG90IDwtIHBsb3RfbmFzYShsb25nX25hc2FfZGF0YSAlPiUgZmlsdGVyKHN1Z2dlc3Rpb25zX3R5cGUgPT0gQkFSX1NVR0dFU1RJT05fVFlQRSkpCgpnZ3NhdmUoCiAgZ3JhcGhfcGF0aCgibmFzYS10bHgtYmFyLnBkZiIpLAogIHBsb3QgPSBiYXJfbmFzYV9wbG90LAogIHdpZHRoID0gRlVMTF9XSURUSCwKICBoZWlnaHQgPSAoRlVMTF9XSURUSCAtIDIwKSAvIEdPTERFTl9SQVRJTywKICB1bml0cyA9ICJtbSIsCiAgZGV2aWNlID0gY2Fpcm9fcGRmCikKCmlubGluZV9uYXNhX3Bsb3QKYGBgCgpgYGB7ciwgZGV2PSJzdmdsaXRlIn0KZm9yKHEgaW4gbGV2ZWxzKGxvbmdfbmFzYV9kYXRhJG5hc2FfcXVlc3Rpb24pKXsKICBuYXNhX3FfZGF0YSA8LSBsb25nX25hc2FfZGF0YSAlPiUKICAgIGZpbHRlcihuYXNhX3F1ZXN0aW9uID09IHEgJgogICAgICAgICAgICAgc3VnZ2VzdGlvbnNfdHlwZSA9PSBCQVJfU1VHR0VTVElPTl9UWVBFKSAlPiUgCiAgICBzZWxlY3QocGFydGljaXBhbnQsIG5hc2FfdmFsdWUsIGFjY3VyYWN5LCBrZXl0aW1lLCBuYXNhX3F1ZXN0aW9uKQogIG1fbmFzYSA9IGFydChuYXNhX3ZhbHVlIH4ga2V5dGltZSAqIGFjY3VyYWN5LCBkYXRhPW5hc2FfcV9kYXRhKQoKICBtZXNzYWdlKHBhc3RlKCdcbj09PT09PT09PT09PT09JywgcSwgJyAoQkFSKSA9PT09PT09PT09PT09PScpKQogIAogIG1lc3NhZ2UoJy0tLS0tLS0tIGFub3ZhIC0tLS0tLS0tJykKICBtIDwtIGFub3ZhKG1fbmFzYSwgdHlwZSA9IDIpCiAgcHJpbnQobSkKICBhbm92YV90aWJsZSA8LSBhbm92YV90b190aWJibGUobSkgJT4lCiAgICBtdXRhdGUocXVlc3Rpb24gPSBxLAogICAgICAgICAgIHNjYWxlID0gIm5hc2EtdGx4IiwKICAgICAgICAgICBzdWdnZXN0aW9ucyA9ICJiYXIiKQogIAogIGlmIChleGlzdHMoImFsbF9hbm92YXMiKSkgewogICAgYWxsX2Fub3ZhcyA8LSB1bmlvbl9hbGwoYWxsX2Fub3ZhcywgYW5vdmFfdGlibGUpCiAgfSBlbHNlIHsKICAgIGFsbF9hbm92YXMgPC0gYW5vdmFfdGlibGUKICB9CiAgCiAgbWVzc2FnZSgnLS0tLS0tLS0gY29udHJhc3RzIGtleSBzdHJva2UgKEJBUikgLS0tLS0tLS0nKQogIHByaW50KGVtbWVhbnMoYXJ0bG0obV9uYXNhLCAia2V5dGltZSIpLCBwYWlyd2lzZSB+IGtleXRpbWUpKQoKICBtZXNzYWdlKCctLS0tLS0tLSBjb250cmFzdHMgYWNjdXJhY3kgKEJBUikgLS0tLS0tLS0nKQogIHByaW50KGVtbWVhbnMoYXJ0bG0obV9uYXNhLCAiYWNjdXJhY3kiKSwgcGFpcndpc2UgfiBhY2N1cmFjeSkpCn0KYGBgCgojIyBJbmxpbmUgdnMgQmFyCgpgYGB7cn0KZm9yKHEgaW4gbGV2ZWxzKGxvbmdfbmFzYV9kYXRhJG5hc2FfcXVlc3Rpb24pKSB7CiAgbmFzYV9xX2RhdGEgPC0gbG9uZ19uYXNhX2RhdGEgJT4lCiAgICBmaWx0ZXIobmFzYV9xdWVzdGlvbiA9PSBxKSAlPiUKICAgIHNlbGVjdCgKICAgICAgcGFydGljaXBhbnQsCiAgICAgIG5hc2FfdmFsdWUsCiAgICAgIGFjY3VyYWN5LAogICAgICBrZXl0aW1lLAogICAgICBuYXNhX3F1ZXN0aW9uLAogICAgICBzdWdnZXN0aW9uc190eXBlCiAgICApCiAgbV9uYXNhID0gYXJ0KG5hc2FfdmFsdWUgfiBrZXl0aW1lICogYWNjdXJhY3kgKiBzdWdnZXN0aW9uc190eXBlLAogICAgICAgICAgICAgICBkYXRhID0gbmFzYV9xX2RhdGEpCiAgCiAgbWVzc2FnZShwYXN0ZSgnXG49PT09PT09PT09PT09PScsIHEsICc9PT09PT09PT09PT09PScpKQogIAogIG1lc3NhZ2UoJy0tLS0tLS0tIGFub3ZhIC0tLS0tLS0tJykKICBtIDwtIGFub3ZhKG1fbmFzYSwgdHlwZSA9IDIpCiAgcHJpbnQobSkKICAKICBtZXNzYWdlKCctLS0tLS0tLSBjb250cmFzdHMgc3VnZ2VzdGlvbnMgdHlwZSAtLS0tLS0tLScpCiAgcHJpbnQoZW1tZWFucygKICAgIGFydGxtKG1fbmFzYSwgInN1Z2dlc3Rpb25zX3R5cGUiKSwKICAgIHBhaXJ3aXNlIH4gc3VnZ2VzdGlvbnNfdHlwZQogICkpCn0KYGBgCgoKCiMgU3ViamVjdGl2ZSBRdWVzdGlvbm5haXJlCgoKIyMgR3JhcGgKCmBgYHtyLCBkZXY9InN2Z2xpdGUifQpsb25nX3F1ZXN0aW9uc19kYXRhID0gdHJ1c3RlZF9wYXJ0aWNpcGFudHNfcnVucyAlPiUKICBnYXRoZXIoCiAgICBrZXkgPSAicXVlc3Rpb24iLAogICAgdmFsdWUgPSAiYW5zd2VyIiwKICAgIGZhY3Rvcl9rZXkgPSBUUlVFLAogICAgY29udHJvbHNfc2F0aXNmYWN0b3J5LAogICAgc3VnZ2VzdGlvbnNfYWNjdXJhY3ksCiAgICBrZXlib2FyZF91c2VfZWZmaWNpZW5jeSwKICAgIHN1Z2dlc3Rpb25fZGlzdHJhY3Rpb24KICApICU+JQogIG11dGF0ZSgKICAgICMgTWFrZSBzdXJlIGFuc3dlciBpcyBhIGZhY3Rvci4KICAgIGFuc3dlciA9ICBmYWN0b3IoYW5zd2VyLCBsZXZlbHMgPSBBR1JFRU1FTlRfTEVWRUxTLCBvcmRlcmVkID0gVCksCiAgICBudW1fYW5zd2VyID0gYW5zd2VyICU+JQogICAgICByZWNvZGUoCiAgICAgICAgYFN0cm9uZ2x5IGRpc2FncmVlYCA9IC0zLAogICAgICAgIGBEaXNhZ3JlZWAgPSAtMiwKICAgICAgICBgU29tZXdoYXQgZGlzYWdyZWVgID0gLTEsCiAgICAgICAgYE5laXRoZXIgYWdyZWUgbm9yIGRpc2FncmVlYCA9IDAsCiAgICAgICAgYFNvbWV3aGF0IGFncmVlYCA9IDEsCiAgICAgICAgYEFncmVlYCA9IDIsCiAgICAgICAgYFN0cm9uZ2x5IGFncmVlYCA9IDMKICAgICAgKSAlPiUKICAgICAgYXMubnVtZXJpYygpLAogICAgcXVlc3Rpb25fbGFiZWwgPSBxdWVzdGlvbiAlPiUKICAgICAgcmVjb2RlX2ZhY3RvcigKICAgICAgICBzdWdnZXN0aW9uc19hY2N1cmFjeSA9ICJQZXJjZWl2ZWQgQWNjdXJhY3kiLAogICAgICAgIGtleWJvYXJkX3VzZV9lZmZpY2llbmN5ID0gIlBlcmNlaXZlZCBVc2FiaWxpdHkiLAogICAgICAgIGNvbnRyb2xzX3NhdGlzZmFjdG9yeSA9ICJTYXRpc2ZhY3Rpb24iLAogICAgICAgIHN1Z2dlc3Rpb25fZGlzdHJhY3Rpb24gPSAiU3VnZ2VzdGlvbnMnIERpc3J1cHRpdml0eSIsCiAgICAgICAgLm9yZGVyZWQgPSBUUlVFCiAgICAgICkKICApICU+JQogIHNlbGVjdCgKICAgIHBhcnRpY2lwYW50LAogICAgcXVlc3Rpb24sCiAgICBxdWVzdGlvbl9sYWJlbCwKICAgIGFjY3VyYWN5LAogICAgYW5zd2VyLAogICAgbnVtX2Fuc3dlciwKICAgIGtleXRpbWUsCiAgICBzdWdnZXN0aW9uc190eXBlCiAgKQoKCnBsb3RfbGV2ZWxzID0gbGlzdCgKICBTQ0FMRV9DT0xPUl9LRVlfU1RST0tFX0RJU0NSRVRFLAogIFNDQUxFX0ZJTExfS0VZX1NUUk9LRV9ESVNDUkVURSwKICBTQ0FMRV9YX0FDQ1VSQUNZX0RJU0NSRVRFLAogIHNjYWxlX3lfY29udGludW91cygKICAgIGxpbWl0cyA9IGMoLTMsIDMpLAogICAgYnJlYWtzID0gLTM6MywKICAgIGxhYmVscyA9IGxpa2VydF9sZXZlbHMgPC0gYygiU3Ryb25nbHkgZGlzYWdyZWUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJTdHJvbmdseSBhZ3JlZSIpCiAgKSwKICBnZW9tX2JveHBsb3Qob3V0bGllci5zaGFwZSA9IE5BKSwKICB0aGVtZSgKICAgIGF4aXMudGl0bGUueSA9IGVsZW1lbnRfYmxhbmsoKSwKICAgIHN0cmlwLnRleHQgPSBlbGVtZW50X3RleHQobWFyZ2luID0gbWFyZ2luKAogICAgICB0ID0gMCwKICAgICAgbCA9IDAsCiAgICAgIGIgPSAyLAogICAgICByID0gMAogICAgKSksCiAgICBsZWdlbmQucG9zaXRpb24gPSAibm9uZSIsCiAgKQopCgpnZ3Bsb3QoCiAgbG9uZ19xdWVzdGlvbnNfZGF0YSwKICBhZXMoCiAgICB4ID0gYWNjdXJhY3ksCiAgICB5ID0gbnVtX2Fuc3dlciwKICAgIGNvbG9yID0ga2V5dGltZSwKICAgIGZpbGwgPSBrZXl0aW1lCiAgKQopICsgcGxvdF9sZXZlbHMgKwogIGZhY2V0X2dyaWQoY29scyA9IHZhcnMocXVlc3Rpb25fbGFiZWwpLAogICAgICAgICAgICAgcm93cyA9IHZhcnMoc3VnZ2VzdGlvbnNfdHlwZSkpCiMKIwojICAgZ2dzYXZlKAojICAgICBncmFwaF9wYXRoKCJzdWJqZWN0aXZlLnBkZiIpLAojICAgICB3aWR0aCA9IEZVTExfV0lEVEgsCiMgICAgIGhlaWdodCA9IChGVUxMX1dJRFRIIC8gNykgKiBHT0xERU5fUkFUSU8sCiMgICAgIHVuaXRzPSJtbSIsCiMgICAgIGRldmljZT1jYWlyb19wZGYKIyAgICkKYGBgCgoKIyMgU3RhdHMgSW5saW5lIFN1Z2dlc3Rpb25zCgpgYGB7cn0KZm9yKHEgaW4gbGV2ZWxzKGxvbmdfcXVlc3Rpb25zX2RhdGEkcXVlc3Rpb24pKXsKICBxX2RhdGEgPC0gbG9uZ19xdWVzdGlvbnNfZGF0YSAlPiUKICAgIGZpbHRlcihxdWVzdGlvbiA9PSBxICYgc3VnZ2VzdGlvbnNfdHlwZSA9PSBJTkxJTkVfU1VHR0VTVElPTl9UWVBFKQoKICBtX25hc2EgPSBhcnQoYW5zd2VyIH4ga2V5dGltZSAqIGFjY3VyYWN5LCBkYXRhPXFfZGF0YSkKCiAgbWVzc2FnZShwYXN0ZSgnXG49PT09PT09PT09PT09PScsIHEsICcgKElOTElORSkgPT09PT09PT09PT09PT0nKSkKICAKICBtZXNzYWdlKCctLS0tLS0tLSBhbm92YSAtLS0tLS0tLScpCiAgbSA8LSBhbm92YShtX25hc2EsIHR5cGUgPSAyKQogIHByaW50KG0pCiAgYWxsX2Fub3ZhcyA8LSBhbm92YV90b190aWJibGUobSkgJT4lCiAgICBtdXRhdGUocXVlc3Rpb24gPSBxLCBzY2FsZT0icXVlc3Rpb25uYWlyZSIsIHN1Z2dlc3Rpb25zPSJpbmxpbmUiKSAlPiUKICAgIHVuaW9uX2FsbChhbGxfYW5vdmFzKQogIG1lc3NhZ2UoJy0tLS0tLS0tIGNvbnRyYXN0cyBrZXkgc3Ryb2tlIGRlbGF5IChJTkxJTkUpIC0tLS0tLS0tJykKICBwcmludChlbW1lYW5zKGFydGxtKG1fbmFzYSwgImtleXRpbWUiKSwgcGFpcndpc2UgfiBrZXl0aW1lKSkKCiAgbWVzc2FnZSgnLS0tLS0tLS0gY29udHJhc3RzIGFjY3VyYWN5IChJTkxJTkUpIC0tLS0tLS0tJykKICBwcmludChlbW1lYW5zKGFydGxtKG1fbmFzYSwgImFjY3VyYWN5IiksIHBhaXJ3aXNlIH4gYWNjdXJhY3kpKQp9CmBgYAoKIyMgU3RhdHMgQmFyIFN1Z2dlc3Rpb25zCgpgYGB7cn0KZm9yKHEgaW4gbGV2ZWxzKGxvbmdfcXVlc3Rpb25zX2RhdGEkcXVlc3Rpb24pKXsKICBxX2RhdGEgPC0gbG9uZ19xdWVzdGlvbnNfZGF0YSAlPiUKICAgIGZpbHRlcihxdWVzdGlvbiA9PSBxICYKICAgICAgICAgICAgIHN1Z2dlc3Rpb25zX3R5cGUgPT0gQkFSX1NVR0dFU1RJT05fVFlQRSkKCiAgbV9uYXNhID0gYXJ0KGFuc3dlciB+IGtleXRpbWUgKiBhY2N1cmFjeSwgZGF0YT1xX2RhdGEpCgogIG1lc3NhZ2UocGFzdGUoJ1xuPT09PT09PT09PT09PT0nLCBxLCAnIChCQVIpID09PT09PT09PT09PT09JykpCiAgCiAgbWVzc2FnZSgnLS0tLS0tLS0gYW5vdmEgLS0tLS0tLS0nKQogIG0gPC0gYW5vdmEobV9uYXNhLCB0eXBlID0gMikKICBwcmludChtKQogIGFsbF9hbm92YXMgPC0gYW5vdmFfdG9fdGliYmxlKG0pICU+JQogICAgbXV0YXRlKHF1ZXN0aW9uID0gcSwgc2NhbGU9InF1ZXN0aW9ubmFpcmUiLCBzdWdnZXN0aW9ucz0iYmFyIikgJT4lCiAgICB1bmlvbl9hbGwoYWxsX2Fub3ZhcykKICBtZXNzYWdlKCctLS0tLS0tLSBjb250cmFzdHMga2V5IHN0cm9rZSAoQkFSKSAtLS0tLS0tLScpCiAgcHJpbnQoZW1tZWFucyhhcnRsbShtX25hc2EsICJrZXl0aW1lIiksIHBhaXJ3aXNlIH4ga2V5dGltZSkpCgogIG1lc3NhZ2UoJy0tLS0tLS0tIGNvbnRyYXN0cyBhY2N1cmFjeSAoQkFSKSAtLS0tLS0tLScpCiAgcHJpbnQoZW1tZWFucyhhcnRsbShtX25hc2EsICJhY2N1cmFjeSIpLCBwYWlyd2lzZSB+IGFjY3VyYWN5KSkKfQpgYGAKCgojIyBJbmxpbmUgdnMgQmFyCmBgYHtyfQpsb25nX3F1ZXN0aW9uc19kYXRhICU+JQogIGZpbHRlcihxdWVzdGlvbiA9PSAic3VnZ2VzdGlvbl9kaXN0cmFjdGlvbiIpICU+JQogICMgZmlsdGVyKGFjY3VyYWN5ID09IDAuNSkgJT4lCiAgZ3JvdXBfYnkoc3VnZ2VzdGlvbnNfdHlwZSkgJT4lCiAgc3VtbWFyaXNlKAogICAgbl9hZ3JlZSA9IHN1bShhbnN3ZXIgPT0gIlN0cm9uZ2x5IGFncmVlIiB8IGFuc3dlciA9PSAiQWdyZWUiKSwKICAgIHRvdGFsID0gbigpLAogICAgcmF0aW8gPSBuX2FncmVlIC8gdG90YWwKICApCmBgYAoKCmBgYHtyfQpmb3IocSBpbiBsZXZlbHMobG9uZ19xdWVzdGlvbnNfZGF0YSRxdWVzdGlvbikpIHsKICBxX2RhdGEgPC0gbG9uZ19xdWVzdGlvbnNfZGF0YSAlPiUKICAgIGZpbHRlcihxdWVzdGlvbiA9PSBxKQogIAogIG1fbmFzYSA9IGFydChhbnN3ZXIgfiBrZXl0aW1lICogYWNjdXJhY3kgKiBzdWdnZXN0aW9uc190eXBlLAogICAgICAgICAgICAgICBkYXRhID0gcV9kYXRhKQogIAogIG1lc3NhZ2UocGFzdGUoJ1xuPT09PT09PT09PT09PT0nLCBxLCAnPT09PT09PT09PT09PT0nKSkKICAKICBtZXNzYWdlKCctLS0tLS0tLSBhbm92YSAtLS0tLS0tLScpCiAgbSA8LSBhbm92YShtX25hc2EsIHR5cGUgPSAyKQogIHByaW50KG0pCgogIG1lc3NhZ2UoJy0tLS0tLS0tIGNvbnRyYXN0cyBzdWdnZXN0aW9ucyB0eXBlIC0tLS0tLS0tJykKICBwcmludChlbW1lYW5zKAogICAgYXJ0bG0obV9uYXNhLCAic3VnZ2VzdGlvbnNfdHlwZSIpLAogICAgcGFpcndpc2UgfiBzdWdnZXN0aW9uc190eXBlCiAgKSkKICAKfQpgYGAKIyBFeHBvcnQKCmBgYHtyIGluY2x1ZGU9RkFMU0V9CndyaXRlX2NzdihhbGxfYW5vdmFzLCAnYW5vdmFzLmNzdicpCmBgYAoKW0dvIGJhY2sgdG8gbWFpbl0oLi9tYWluLm5iLmh0bWwpCg==