Word-Suggestions: Device Subjective Results

library(tidyverse)
library(ARTool)
library(emmeans)
library(patchwork)
library(svglite)
source("./device-setup.R")

nasa_labels = rep('', 20)
nasa_labels[20] = "high"
nasa_labels[1] = "low"

measured_runs <- read_device_runs(measured_only=TRUE)

# Store all ANOVA results to automatically create the LaTeX table.
all_anovas <- tibble(
  term = character(),
  df = numeric(),
  df_res = numeric(),
  f = numeric(),
  p = numeric(),
  question = character(),
  scale = character(),
  suggestions = character(),
  no_pairwise = logical()
)

1 Nasa TLX

1.1 Graph

long_nasa_data <- measured_runs %>%
  gather(
    key = "nasa_question",
    value = "nasa_value",
    factor_key = TRUE,
    effort,
    frustration,
    performance,
    mental_demand,
    physical_demand,
    temporal_demand
  ) %>%
  mutate(
    nasa_question = nasa_question %>% as_factor(),
    question_label = nasa_question %>%
      recode(
        mental_demand = "Mental Demand",
        physical_demand = "Physical Demand",
        temporal_demand = "Temporal Demand",
        performance = "Overall Performance",
        effort = "Effort",
        frustration = "Frustration Level"
      )
  ) %>%
  select(participant,
         device,
         accuracy,
         nasa_question,
         question_label,
         nasa_value)


pd <- position_dodge(0.1)
nasa_plot <- ggplot(long_nasa_data, aes(x = accuracy,  y = nasa_value, color=device)) +
  expand_limits(y = c(5, 100)) +
  SCALE_X_ACCURACY_DISCRETE +
  scale_y_continuous("", breaks = seq(5, 100, 5), labels = nasa_labels) +
  geom_boxplot(outlier.shape = NA) +
  theme(
    axis.title.y = element_blank(),
    legend.position = "none",
    strip.text = element_text(margin=margin(t=0, l=0, b=2, r=0))
  ) +
  scale_color_manual("Device", values = DEVICE_COLORS) +
  facet_grid(cols = vars(question_label))

nasa_plot

1.2 Stats

for(q in levels(long_nasa_data$nasa_question)){
  nasa_q_data <- long_nasa_data %>%
    filter(nasa_question == q) %>% 
    select(participant, nasa_value, accuracy, device, nasa_question)
  m_nasa = art(nasa_value ~ device * accuracy, data=nasa_q_data)

  message(paste('\n==============', toupper(q), '=============='))
  
  message('-------- anova --------')
  m <- anova(m_nasa, type = 2)
  print(m)
  all_anovas <- anova_to_tibble(m) %>%
    mutate(question = q, scale="nasa-tlx", suggestions="bar", no_pairwise=F) %>%
    union_all(all_anovas)
  
  message('-------- contrasts device --------')
  print(emmeans(artlm(m_nasa, "device"), pairwise ~ device))

  message('-------- contrasts accuracy --------')
  print(emmeans(artlm(m_nasa, "accuracy"), pairwise ~ accuracy))
  message("")
}

============== EFFORT ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(nasa_value)

                  Df Df.res F value  Pr(>F)  
1 device           2     99 3.56117 0.03210 *
2 accuracy         2     99 0.40462 0.66833  
3 device:accuracy  4     99 0.47224 0.75599  
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   43.1 5.24 99     32.7     53.5
 tablet    60.8 5.24 99     50.4     71.2
 phone     59.6 5.24 99     49.2     70.0

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet   -17.69 7.41 99  -2.389  0.0488
 desktop - phone    -16.47 7.41 99  -2.224  0.0720
 tablet - phone       1.22 7.41 99   0.165  0.9851

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean  SE df lower.CL upper.CL
 0.1        53.8 5.4 99     43.0     64.5
 0.5        58.2 5.4 99     47.5     69.0
 0.9        51.5 5.4 99     40.8     62.2

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5    -4.50 7.64 99  -0.589  0.8264
 accuracy0.1 - accuracy0.9     2.25 7.64 99   0.294  0.9534
 accuracy0.5 - accuracy0.9     6.75 7.64 99   0.883  0.6520

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== FRUSTRATION ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(nasa_value)

                  Df Df.res F value    Pr(>F)    
1 device           2     99 8.01366 0.0005947 ***
2 accuracy         2     99 0.94996 0.3902541    
3 device:accuracy  4     99 0.75331 0.5581520    
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   38.1 5.03 99     28.1     48.0
 tablet    63.0 5.03 99     53.0     73.0
 phone     62.5 5.03 99     52.5     72.5

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet    -24.9 7.12 99  -3.502  0.0020
 desktop - phone     -24.4 7.12 99  -3.431  0.0025
 tablet - phone        0.5 7.12 99   0.070  0.9973

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        49.0 5.37 99     38.3     59.7
 0.5        59.4 5.37 99     48.8     70.1
 0.9        55.1 5.37 99     44.4     65.7

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5   -10.42 7.59 99  -1.372  0.3595
 accuracy0.1 - accuracy0.9    -6.08 7.59 99  -0.801  0.7032
 accuracy0.5 - accuracy0.9     4.33 7.59 99   0.571  0.8360

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== PERFORMANCE ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(nasa_value)

                  Df Df.res F value   Pr(>F)  
1 device           2     99  1.8751 0.158746  
2 accuracy         2     99  3.0533 0.051672 .
3 device:accuracy  4     99  3.4925 0.010348 *
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   46.8 5.31 99     36.3     57.3
 tablet    55.4 5.31 99     44.9     66.0
 phone     61.3 5.31 99     50.7     71.8

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet    -8.62 7.51 99  -1.148  0.4870
 desktop - phone    -14.46 7.51 99  -1.925  0.1371
 tablet - phone      -5.83 7.51 99  -0.777  0.7183

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        61.0 5.26 99     50.5     71.4
 0.5        58.6 5.26 99     48.1     69.0
 0.9        44.0 5.26 99     33.5     54.4

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5     2.42 7.44 99   0.325  0.9436
 accuracy0.1 - accuracy0.9    17.00 7.44 99   2.284  0.0628
 accuracy0.5 - accuracy0.9    14.58 7.44 99   1.959  0.1278

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== MENTAL_DEMAND ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(nasa_value)

                  Df Df.res F value   Pr(>F)  
1 device           2     99  2.8031 0.065439 .
2 accuracy         2     99  1.5600 0.215264  
3 device:accuracy  4     99  0.3061 0.873282  
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   45.4 5.28 99     34.9     55.9
 tablet    63.0 5.28 99     52.6     73.5
 phone     55.1 5.28 99     44.6     65.6

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet   -17.64 7.46 99  -2.364  0.0519
 desktop - phone     -9.69 7.46 99  -1.299  0.3990
 tablet - phone       7.94 7.46 99   1.065  0.5382

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        46.9 5.34 99     36.3     57.5
 0.5        59.3 5.34 99     48.7     69.9
 0.9        57.3 5.34 99     46.7     67.9

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5   -12.44 7.55 99  -1.647  0.2308
 accuracy0.1 - accuracy0.9   -10.39 7.55 99  -1.375  0.3577
 accuracy0.5 - accuracy0.9     2.06 7.55 99   0.272  0.9600

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== PHYSICAL_DEMAND ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(nasa_value)

                  Df Df.res F value     Pr(>F)    
1 device           2     99  8.6432 0.00034695 ***
2 accuracy         2     99  3.0363 0.05250775   .
3 device:accuracy  4     99  1.2636 0.28945656    
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   37.6 5.01 99     27.7     47.5
 tablet    61.3 5.01 99     51.4     71.3
 phone     64.6 5.01 99     54.6     74.5

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet   -23.72 7.08 99  -3.351  0.0032
 desktop - phone    -26.94 7.08 99  -3.807  0.0007
 tablet - phone      -3.22 7.08 99  -0.455  0.8922

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        48.1 5.26 99     37.6     58.5
 0.5        65.0 5.26 99     54.6     75.4
 0.9        50.4 5.26 99     40.0     60.9

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5   -16.94 7.44 99  -2.276  0.0639
 accuracy0.1 - accuracy0.9    -2.39 7.44 99  -0.321  0.9448
 accuracy0.5 - accuracy0.9    14.56 7.44 99   1.955  0.1288

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== TEMPORAL_DEMAND ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(nasa_value)

                  Df Df.res F value   Pr(>F)  
1 device           2     99 2.16419 0.120246  
2 accuracy         2     99 2.75922 0.068217 .
3 device:accuracy  4     99 0.54755 0.701216  
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean  SE df lower.CL upper.CL
 desktop   48.7 5.3 99     38.1     59.2
 tablet    63.3 5.3 99     52.8     73.9
 phone     51.5 5.3 99     41.0     62.0

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet   -14.69 7.49 99  -1.961  0.1273
 desktop - phone     -2.85 7.49 99  -0.380  0.9235
 tablet - phone      11.85 7.49 99   1.581  0.2584

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        46.9 5.27 99     36.4     57.3
 0.5        64.1 5.27 99     53.6     74.5
 0.9        52.6 5.27 99     42.1     63.0

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5   -17.19 7.46 99  -2.305  0.0597
 accuracy0.1 - accuracy0.9    -5.68 7.46 99  -0.762  0.7273
 accuracy0.5 - accuracy0.9    11.51 7.46 99   1.544  0.2751

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

all_anovas <- all_anovas %>%
  mutate(no_pairwise = no_pairwise |
           question == "effort" & scale == "nasa-tlx" & term ==
           "device")

2 Subjective Questionnaire

2.1 Graph

long_questions_data = measured_runs %>%
  gather(
    key = "question",
    value = "answer",
    factor_key = TRUE,
    controls_satisfactory,
    suggestions_accuracy,
    keyboard_use_efficiency,
    suggestion_distraction
  ) %>%
  mutate(
    # Make sure answer is a factor.
    answer =  factor(answer, levels=AGREEMENT_LEVELS, ordered=T),
    num_answer = answer %>%
      recode(
        `Strongly disagree` = -3,
        `Disagree` = -2,
        `Somewhat disagree` = -1,
        `Neither agree nor disagree` = 0,
        `Somewhat agree` = 1,
        `Agree` = 2,
        `Strongly agree` = 3
      ) %>%
      as.numeric(),
    question_label = question %>%
      recode_factor(
        suggestions_accuracy = "Perceived Accuracy",
        keyboard_use_efficiency = "Perceived Keyboard Efficiency",
        controls_satisfactory = "Satisfaction",
        suggestion_distraction = "Suggestions' Disruptivity",
        .ordered = TRUE
      ),
    device = device %>% recode_factor(laptop = "desktop")
  ) %>%
  select(participant, question,question_label, accuracy, answer, num_answer, device)



questionnaire_plot <- ggplot(
  long_questions_data,
  aes(
    x = accuracy,
    y = num_answer,
    color = device,
  )
) +
  facet_grid(cols=vars(question_label)) +
  scale_color_manual("Device", values = DEVICE_COLORS) +
  SCALE_X_ACCURACY_DISCRETE +
  scale_y_continuous(limits=c(-3, 3), breaks=-3:3, labels=likert_levels <- c(
    "strongly disagree",
    "",
    "",
    "",
    "",
    "", 
    "strongly agree"
)) +
  geom_boxplot(outlier.shape = NA) +
  # facet_wrap(vars(suggestions_type)) +
  # custom_line(position=pd) +
  # custom_pointrange(position=pd) +
  # THEME_BOTTOM_LEGEND +
  theme(
    axis.title.y = element_blank(),
    strip.text = element_text(margin=margin(t=0, l=0, b=2, r=0)),
    # legend.position = "none",
    # We need to add a right margin to prevent the legend to stick out, and a negative top
    # margin because I have not been able to bring that legend close to the plot in any other way.
    # plot.margin = margin(l =0.75, r=0.75, unit="cm")
  )

questionnaire_plot

2.2 Stats

for(q in levels(long_questions_data$question)){
  q_data <- long_questions_data %>% filter(question == q)

  m_nasa = art(answer ~ device * accuracy, data=q_data)

  message(paste('\n==============', toupper(q), '=============='))
  
  message('-------- anova --------')
  m <- anova(m_nasa, type = 2)
  print(m)
  all_anovas <- anova_to_tibble(m) %>%
    mutate(question = q, scale="questionnaire", suggestions="bar", no_pairwise=F) %>%
    union_all(all_anovas)
  message('-------- contrasts device --------')
  print(emmeans(artlm(m_nasa, "device"), pairwise ~ device))

  message('-------- contrasts accuracy --------')
  print(emmeans(artlm(m_nasa, "accuracy"), pairwise ~ accuracy))
  message("")
}

============== CONTROLS_SATISFACTORY ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(answer)

                  Df Df.res F value     Pr(>F)    
1 device           2     99  5.3844 0.00602898  **
2 accuracy         2     99  7.6095 0.00084316 ***
3 device:accuracy  4     99  1.7940 0.13603901    
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   68.0 5.15 99     57.8     78.2
 tablet    45.3 5.15 99     35.1     55.5
 phone     50.2 5.15 99     40.0     60.4

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet    22.72 7.28 99   3.120  0.0066
 desktop - phone     17.78 7.28 99   2.441  0.0430
 tablet - phone      -4.94 7.28 99  -0.679  0.7763

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        43.0 5.03 99     33.0     53.0
 0.5        50.6 5.03 99     40.6     60.5
 0.9        69.9 5.03 99     59.9     79.9

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5    -7.53 7.11 99  -1.059  0.5420
 accuracy0.1 - accuracy0.9   -26.89 7.11 99  -3.781  0.0008
 accuracy0.5 - accuracy0.9   -19.36 7.11 99  -2.722  0.0207

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== SUGGESTIONS_ACCURACY ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(answer)

                  Df Df.res  F value   Pr(>F)    
1 device           2     99  0.31647 0.729453    
2 accuracy         2     99 74.36476  < 2e-16 ***
3 device:accuracy  4     99  2.09003 0.087773   .
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   54.6 5.36 99     44.0     65.2
 tablet    51.4 5.36 99     40.8     62.1
 phone     57.5 5.36 99     46.8     68.1

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet     3.18 7.58 99   0.420  0.9076
 desktop - phone     -2.85 7.58 99  -0.376  0.9252
 tablet - phone      -6.03 7.58 99  -0.795  0.7069

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        25.1 3.43 99     18.3     31.9
 0.5        54.1 3.43 99     47.3     60.9
 0.9        84.2 3.43 99     77.4     91.0

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5    -29.0 4.84 99  -5.986  <.0001
 accuracy0.1 - accuracy0.9    -59.1 4.84 99 -12.195  <.0001
 accuracy0.5 - accuracy0.9    -30.1 4.84 99  -6.209  <.0001

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== KEYBOARD_USE_EFFICIENCY ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(answer)

                  Df Df.res  F value     Pr(>F)    
1 device           2     99 22.08550 1.1725e-08 ***
2 accuracy         2     99  0.16167    0.85095    
3 device:accuracy  4     99  0.22892    0.92159    
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   78.9 4.51 99     70.0     87.9
 tablet    43.1 4.51 99     34.2     52.1
 phone     41.4 4.51 99     32.5     50.4

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet    35.83 6.38 99   5.621  <.0001
 desktop - phone     37.50 6.38 99   5.882  <.0001
 tablet - phone       1.67 6.38 99   0.261  0.9630

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        52.9 5.37 99     42.2     63.6
 0.5        53.6 5.37 99     43.0     64.3
 0.9        57.0 5.37 99     46.3     67.6

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate  SE df t.ratio p.value
 accuracy0.1 - accuracy0.5   -0.736 7.6 99  -0.097  0.9948
 accuracy0.1 - accuracy0.9   -4.056 7.6 99  -0.534  0.8550
 accuracy0.5 - accuracy0.9   -3.319 7.6 99  -0.437  0.9003

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates 

============== SUGGESTION_DISTRACTION ==============

-------- anova --------

Analysis of Variance of Aligned Rank Transformed Data

Table Type: Anova Table (Type II tests) 
Model: No Repeated Measures (lm)
Response: art(answer)

                  Df Df.res F value     Pr(>F)    
1 device           2     99  2.6566 0.07518672   .
2 accuracy         2     99  8.7328 0.00032147 ***
3 device:accuracy  4     99  1.2517 0.29421863    
---
Signif. codes:   0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 

-------- contrasts device --------

NOTE: Results may be misleading due to involvement in interactions

$emmeans
 device  emmean   SE df lower.CL upper.CL
 desktop   60.9 5.28 99     50.4     71.4
 tablet    57.9 5.28 99     47.4     68.3
 phone     44.7 5.28 99     34.2     55.2

Results are averaged over the levels of: accuracy 
Confidence level used: 0.95 

$contrasts
 contrast         estimate   SE df t.ratio p.value
 desktop - tablet     3.06 7.47 99   0.409  0.9119
 desktop - phone     16.19 7.47 99   2.169  0.0816
 tablet - phone      13.14 7.47 99   1.760  0.1885

Results are averaged over the levels of: accuracy 
P value adjustment: tukey method for comparing a family of 3 estimates 

-------- contrasts accuracy --------
NOTE: Results may be misleading due to involvement in interactions

$emmeans
 accuracy emmean   SE df lower.CL upper.CL
 0.1        62.9 4.97 99     53.0     72.7
 0.5        63.1 4.97 99     53.2     73.0
 0.9        37.5 4.97 99     27.7     47.4

Results are averaged over the levels of: device 
Confidence level used: 0.95 

$contrasts
 contrast                  estimate   SE df t.ratio p.value
 accuracy0.1 - accuracy0.5    -0.25 7.03 99  -0.036  0.9993
 accuracy0.1 - accuracy0.9    25.33 7.03 99   3.601  0.0014
 accuracy0.5 - accuracy0.9    25.58 7.03 99   3.637  0.0013

Results are averaged over the levels of: device 
P value adjustment: tukey method for comparing a family of 3 estimates