Meta-regression weighted gamlss models

In this script, I perform meta-analyses for the models and test the hypotheses

rm(list=ls())
library(meta)
library(metafor)
library(dplyr)
library(kableExtra)
library(fastDummies)
library(modelsummary)
library(broom.mixed)

set.seed(1)

Forest plots

# With the final data
load("./data/meta_analysis/total_reg_results_gam_w_new.RData")

# Step 1. Estimate the empty model without moderators to make the forest plots. 
model_step_1_att <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              method = "ML",
              test = "knha")

model_step_1_att

# I want to assign different colors to the different types of dependent variables, so that one can easily distinguish 
# the types of attitudes by colors in the forest plot
total_reg_results$colour <- NA
total_reg_results$colour[total_reg_results$attitude_cat=="cognitive"] <- "#2980B9" # blue
total_reg_results$colour[total_reg_results$attitude_cat=="affective"] <- "#CC5279" # pink
total_reg_results$colour[total_reg_results$attitude_cat=="behavioral"] <- "#B0CC52" # green

colour.palette <- as.vector(total_reg_results$colour)

# Give ambiguous variables a different shape
shape <- rep(3, nrow(total_reg_results))
shape[total_reg_results$ambiguous == "Yes"] <- 1

# Make a forest plot (has to be done outside codechunk, otherwise it doesn't save)

# Now plot the model with the 4 pooled effects
# The argument order = "yi" puts the effects in order from negative to positive
# If I don't use the brackets, Rmarkdown doesn't recognize the plot

{png <- forest(model_step_1_att, , at=c(-0.4, 0, 0.4), colout = colour.palette, col = "#ACBFD6", annotate = TRUE, header = "Forest plot time effect attitudes", slab = total_reg_results$dep_var_name, pch = shape,fonts = "Times New Roman", ylim = c(-4.5, 75), cex = 0.25, mlab = "Pooled estimate total", order = "yi", ilab = mean_year, ilab.xpos = -4, xlim = c(-2, 2), width = 0.0001)
}

# The plot can only be saved outside of the codechunk, hence the code underneath

#Mean attitudes png(file = “./output/forestplot_neg_to_pos_weight_NEW_DECEMBER.png”, width = 2800, height = 2400, res = 300)

{png <- forest(model_step_1_att, , at=c(-0.4, 0, 0.4), colout = colour.palette, col = “#ACBFD6”, annotate = TRUE, header = “Forest plot time effect attitudes”, slab = total_reg_results$dep_var_name, pch = shape,fonts = “Times New Roman”, ylim = c(-4.5, 75), cex = 0.25, mlab = “Pooled estimate total”, order = “yi”, ilab = mean_year, ilab.xpos = -4, xlim = c(-2, 2), width = 0.0001) }

dev.off()

# Repeat step 1, but then for the polarization (variance)
model_step_1_var <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              method = "ML",
              test = "knha")

model_step_1_var

## 
## Random-Effects Model (k = 71; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of total heterogeneity): 0.0009 (SE = 0.0002)
## tau (square root of estimated tau^2 value):      0.0300
## I^2 (total heterogeneity / total variability):   98.47%
## H^2 (total variability / sampling variability):  65.49
## 
## Test for Heterogeneity:
## Q(df = 70) = 1445.6155, p-val < .0001
## 
## Model Results:
## 
## estimate      se     tval  df    pval    ci.lb    ci.ub    
##  -0.0093  0.0039  -2.4209  70  0.0181  -0.0170  -0.0016  * 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

{png <- forest(model_step_1_var, at=c(-1, 0, 1), colout = colour.palette, col = "#ACBFD6", annotate = TRUE, header = "Forest plot time effect polarization", slab = total_reg_results$dep_var_name, pch = 23,fonts = "Times", cex = 0.25, ylim = c(-4.5, 75), mlab = "Pooled estimate total", order = "yi", ilab = mean_year, ilab.xpos = -4, xlim = c(-2, 2), width = 0.0001)
}

# And also make the forest plot for the variance

#Polarization attitudes png(file = “./output/forestplot_variance_NEW_DECEMBER.png”, width = 2800, height = 2400, res = 300)

{png <- forest(model_step_1_var, at=c(-0.3, 0, 0.3), colout = colour.palette, col = “#ACBFD6”, annotate = TRUE, header = “Forest plot time effect polarization”, slab = total_reg_results$dep_var_name, pch = shape,fonts = “Times”, cex = 0.5, ylim = c(-4.5, 75), mlab = “Pooled estimate total”, order = “yi”, ilab = mean_year, ilab.xpos = -4, xlim = c(-2, 2)) }

dev.off()

Step 2 until 4 meta-regression

# Check funnel plot asymmetry
funnel(model_step_1_att, col = "steelblue", main = "Funnel Plot")

funnel(model_step_1_var, col = "steelblue", main = "Funnel Plot")

beggs_asymm_mu <- cor.test(rank(total_reg_results$mu_time), total_reg_results$mu_time_sd, method = "kendall") # sig
beggs_assym_sigma <- cor.test(rank(total_reg_results$sig_time), total_reg_results$sig_time_sd, method = "kendall") # not sig

egger_mu <- regtest(total_reg_results$mu_time, total_reg_results$mu_time_sd) # not sig
egger_sig <- regtest(total_reg_results$sig_time, total_reg_results$sig_time_sd) # not sig

duval_tweedie_mu <- trimfill(total_reg_results$mu_time, total_reg_results$mu_time_sd) # insig
duval_tweedie_sig <- trimfill(total_reg_results$sig_time, total_reg_results$sig_time_sd) # insig

#Step 2. Empty model with meta-level indicators, first only with mean year. 
# First I have to give mean_year a meaning
table(total_reg_results$mean_year)
total_reg_results$mean_year_centered <- total_reg_results$mean_year - mean(total_reg_results$mean_year)

#From the plot we saw that behavioral shows something else than the other 2, so set that as the reference category.
table(total_reg_results$attitude_cat)
total_reg_results$attitude_cat <- factor(total_reg_results$attitude_cat)
total_reg_results$attitude_cat <- relevel(total_reg_results$attitude_cat, ref = "behavioral") # Other ref cats also insig for mean, for variance beh as ref cat sig
 
model_step_2_att <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat,
              method = "ML",
              test = "knha")

summary(model_step_2_att)

# Save dataset with new variables

#save(total_reg_results, file= "./data/meta_analysis/total_reg_results_gam_w_new.RData" )

# Step 2 variance
#Step 2. Empty model with meta-level indicators
model_step_2_var <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat, # cogn as ref cat, sig
              method = "ML",
              test = "knha")

model_step_2_var

# Step 3. Model with 2 meta-level vars variables
model_step_3_att <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered,
              method = "ML",
              test = "knha")

model_step_3_att

# Step 4. Model with all metalevel variables
# First check the correlation between the numeric vars
total_reg_results[,c("pec_miss", "or_scale", "national", "mean_year_centered")] %>% cor() # low correlations

model_step_4_att <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "ML",
              test = "knha")

model_step_4_att

# Step 3. Variance
model_step_3_var <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              mods = ~  attitude_cat + mean_year_centered,
              method = "ML",
              test = "knha")

model_step_3_var

# Step 4. Variance
model_step_4_var <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              mods = ~  attitude_cat +  mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "ML",
              test = "knha")

model_step_4_var

Put the results in kable table

models <- list("M1" = model_step_1_att, "M2" = model_step_2_att, "M3" = model_step_3_att, "M4" = model_step_4_att, "M1" =  model_step_1_var, "M2"= model_step_2_var, "M3"=  model_step_3_var, "M4" = model_step_4_var)

modelsummary(models, output = "kableExtra", statistic = 'std.error', stars = TRUE, shape = term ~ model + statistic, title = "
Table 3. Meta-regression on climate change attitudes and polarization.", fmt = fmt_statistic(estimate = 3, std.error =3), coef_rename = c("overall" ="Intercept", 
           "intercept" = "Intercept", 
           "attitude_cataffective" = "Affective attitude (ref = beh)",
           "attitude_catcognitive" = "Cognitive attitude", 
           "mean_year_centered" =   "Mean year centered", 
           "ambiguousYes" = "Ambiguous (ref = no)",
           "national" = "National (ref = no)", 
           "or_scale" = "Original scale", 
            "pec_miss" = "Perc. missings", 
           "dataESS"= "ESS (ref = EB)", 
           "dataEVS"= "EVS", 
           "dataISSP" = "ISSP", 
           "dataLISS"= "LISS", 
           "dataMOT" = "MOT", 
           "nr_waves" = "Nr. of waves"), gof_map = NA) %>%
  kable_classic_2(html_font = "Times", fixed_thead = T, full_width = F) %>%
   add_header_above(c(" " = 1, "Mu" = 8, "Sigma" = 8))%>%
    save_kable("./output/meta_regression_table_NEW_JANUARY.html")

Fixed effects

# Mean attitudes
# Step 1
model_step_1_att_f <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              method = "FE",
              test = "z")

model_step_1_att_f

## 
## Fixed-Effects Model (k = 71)
## 
## I^2 (total heterogeneity / total variability):   98.34%
## H^2 (total variability / sampling variability):  60.23
## 
## Test for Heterogeneity:
## Q(df = 70) = 4216.1447, p-val < .0001
## 
## Model Results:
## 
## estimate      se    zval    pval   ci.lb   ci.ub      
##   0.0026  0.0006  4.2294  <.0001  0.0014  0.0038  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_att_f <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat,
              method = "FE",
              test = "z")

summary(model_step_2_att_f)

## 
## Fixed-Effects with Moderators Model (k = 71)
## 
##     logLik    deviance         AIC         BIC        AICc   
## -1327.7165   3179.7014   2661.4329   2668.2209   2661.7911   
## 
## I^2 (residual heterogeneity / unaccounted variability): 97.86%
## H^2 (unaccounted variability / sampling variability):   46.76
## R^2 (amount of heterogeneity accounted for):            22.36%
## 
## Test for Residual Heterogeneity:
## QE(df = 68) = 3179.7014, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 1036.4432, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                 -0.0244  0.0011  -22.4108  <.0001  -0.0265  -0.0223  *** 
## attitude_cataffective    0.0848  0.0042   20.2762  <.0001   0.0766   0.0930  *** 
## attitude_catcognitive    0.0383  0.0013   28.7544  <.0001   0.0357   0.0409  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_att_f <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered,
              method = "FE",
              test = "z")

model_step_3_att_f

## 
## Fixed-Effects with Moderators Model (k = 71)
## 
## I^2 (residual heterogeneity / unaccounted variability): 97.76%
## H^2 (unaccounted variability / sampling variability):   44.61
## R^2 (amount of heterogeneity accounted for):            25.94%
## 
## Test for Residual Heterogeneity:
## QE(df = 67) = 2988.6381, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## QM(df = 3) = 1227.5066, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se     zval    pval    ci.lb    ci.ub      
## intrcpt                 -0.0064  0.0017  -3.7611  0.0002  -0.0097  -0.0031  *** 
## attitude_cataffective    0.0565  0.0047  12.1260  <.0001   0.0473   0.0656  *** 
## attitude_catcognitive    0.0305  0.0014  21.1418  <.0001   0.0277   0.0333  *** 
## mean_year_centered       0.0016  0.0001  13.8226  <.0001   0.0014   0.0019  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_4_att_f <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "FE",
              test = "z")

## Warning: 9 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_att_f

## 
## Fixed-Effects with Moderators Model (k = 62)
## 
## I^2 (residual heterogeneity / unaccounted variability): 97.93%
## H^2 (unaccounted variability / sampling variability):   48.25
## R^2 (amount of heterogeneity accounted for):            27.72%
## 
## Test for Residual Heterogeneity:
## QE(df = 48) = 2316.0511, p-val < .0001
## 
## Test of Moderators (coefficients 2:14):
## QM(df = 13) = 1756.0553, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                  0.0247  0.0057    4.3210  <.0001   0.0135   0.0360  *** 
## attitude_cataffective    0.0226  0.0078    2.8994  0.0037   0.0073   0.0378   ** 
## attitude_catcognitive    0.0203  0.0017   11.9211  <.0001   0.0170   0.0236  *** 
## mean_year_centered       0.0009  0.0002    4.1850  <.0001   0.0005   0.0013  *** 
## ambiguousYes             0.0033  0.0018    1.8281  0.0675  -0.0002   0.0068    . 
## national                 0.0193  0.0232    0.8345  0.4040  -0.0261   0.0647      
## or_scale                 0.0120  0.0006   21.2945  <.0001   0.0109   0.0131  *** 
## pec_miss                -0.0294  0.0543   -0.5423  0.5876  -0.1359   0.0770      
## dataESS                 -0.0306  0.0077   -3.9808  <.0001  -0.0456  -0.0155  *** 
## dataEVS                 -0.0426  0.0039  -10.8992  <.0001  -0.0502  -0.0349  *** 
## dataISSP                -0.0405  0.0027  -15.2193  <.0001  -0.0457  -0.0353  *** 
## dataLISS                -0.0216  0.0261   -0.8268  0.4083  -0.0728   0.0296      
## dataMOT                 -0.0598  0.0233   -2.5675  0.0102  -0.1054  -0.0141    * 
## nr_waves                -0.0184  0.0013  -14.3289  <.0001  -0.0209  -0.0159  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Fixed effects variance
model_step_1_var_f <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              method = "FE",
              test = "z")

model_step_1_var_f

## 
## Fixed-Effects Model (k = 71)
## 
## I^2 (total heterogeneity / total variability):   95.16%
## H^2 (total variability / sampling variability):  20.65
## 
## Test for Heterogeneity:
## Q(df = 70) = 1445.6155, p-val < .0001
## 
## Model Results:
## 
## estimate      se     zval    pval    ci.lb    ci.ub      
##  -0.0019  0.0004  -4.3279  <.0001  -0.0027  -0.0010  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_var_f <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              mods = ~ attitude_cat, # cogn as ref cat, sig
              method = "FE",
              test = "z")

model_step_2_var_f

## 
## Fixed-Effects with Moderators Model (k = 71)
## 
## I^2 (residual heterogeneity / unaccounted variability): 94.55%
## H^2 (unaccounted variability / sampling variability):   18.37
## R^2 (amount of heterogeneity accounted for):            11.07%
## 
## Test for Residual Heterogeneity:
## QE(df = 68) = 1248.8415, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 196.7740, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                  0.0070  0.0008    9.0893  <.0001   0.0055   0.0085  *** 
## attitude_cataffective   -0.0140  0.0030   -4.7456  <.0001  -0.0198  -0.0082  *** 
## attitude_catcognitive   -0.0131  0.0009  -13.9088  <.0001  -0.0149  -0.0112  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_var_f <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              mods = ~  attitude_cat + mean_year_centered,
              method = "FE",
              test = "z")

model_step_3_var_f

## 
## Fixed-Effects with Moderators Model (k = 71)
## 
## I^2 (residual heterogeneity / unaccounted variability): 94.57%
## H^2 (unaccounted variability / sampling variability):   18.42
## R^2 (amount of heterogeneity accounted for):            10.78%
## 
## Test for Residual Heterogeneity:
## QE(df = 67) = 1234.4369, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## QM(df = 3) = 211.1786, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                  0.0035  0.0012    2.9150  0.0036   0.0011   0.0059   ** 
## attitude_cataffective   -0.0085  0.0033   -2.5916  0.0096  -0.0150  -0.0021   ** 
## attitude_catcognitive   -0.0116  0.0010  -11.3477  <.0001  -0.0136  -0.0096  *** 
## mean_year_centered      -0.0003  0.0001   -3.7953  0.0001  -0.0005  -0.0002  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Step 4. Variance
model_step_4_var_f <- rma(yi = sig_time,
              sei = sig_time_sd,
              data = total_reg_results,
              mods = ~  attitude_cat +  mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "FE",
              test = "z")

## Warning: 9 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_var_f

## 
## Fixed-Effects with Moderators Model (k = 62)
## 
## I^2 (residual heterogeneity / unaccounted variability): 93.96%
## H^2 (unaccounted variability / sampling variability):   16.56
## R^2 (amount of heterogeneity accounted for):            20.53%
## 
## Test for Residual Heterogeneity:
## QE(df = 48) = 794.8834, p-val < .0001
## 
## Test of Moderators (coefficients 2:14):
## QM(df = 13) = 476.2603, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se     zval    pval    ci.lb    ci.ub      
## intrcpt                 -0.0318  0.0040  -7.8436  <.0001  -0.0397  -0.0238  *** 
## attitude_cataffective   -0.0036  0.0055  -0.6458  0.5184  -0.0144   0.0072      
## attitude_catcognitive   -0.0064  0.0012  -5.3266  <.0001  -0.0088  -0.0041  *** 
## mean_year_centered       0.0004  0.0001   2.5138  0.0119   0.0001   0.0007    * 
## ambiguousYes            -0.0013  0.0013  -1.0357  0.3004  -0.0038   0.0012      
## national                -0.0402  0.0164  -2.4566  0.0140  -0.0723  -0.0081    * 
## or_scale                -0.0021  0.0004  -5.2660  <.0001  -0.0029  -0.0013  *** 
## pec_miss                 0.0061  0.0384   0.1577  0.8747  -0.0692   0.0813      
## dataESS                  0.0227  0.0054   4.1794  <.0001   0.0120   0.0333  *** 
## dataEVS                  0.0196  0.0028   7.0826  <.0001   0.0141   0.0250  *** 
## dataISSP                 0.0285  0.0019  15.1495  <.0001   0.0248   0.0322  *** 
## dataLISS                 0.0667  0.0185   3.6110  0.0003   0.0305   0.1030  *** 
## dataMOT                  0.0582  0.0165   3.5346  0.0004   0.0259   0.0905  *** 
## nr_waves                 0.0089  0.0009   9.7863  <.0001   0.0071   0.0107  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Also put the fixed effects in a table
modelsf <- list("M1" = model_step_1_att_f, "M2" = model_step_2_att_f, "M3" = model_step_3_att_f, "M4" = model_step_4_att_f, "M1" =  model_step_1_var_f, "M2"= model_step_2_var_f, "M3"=  model_step_3_var_f, "M4" = model_step_4_var_f)

modelsummary(modelsf, output = "kableExtra", statistic = 'std.error', stars = TRUE, shape = term ~ model + statistic, title = "
Appendix X. Fixed effect meta-regression on climate change attitudes and polarization.", fmt = fmt_statistic(estimate = 3, std.error =3), coef_rename = c("overall" ="Intercept", 
           "intercept" = "Intercept", 
           "attitude_cataffective" = "Affective attitude (ref = beh)",
           "attitude_catcognitive" = "Cognitive attitude", 
           "mean_year_centered" =   "Mean year centered", 
           "ambiguousYes" = "Ambiguous (ref = no)",
           "national" = "National (ref = no)", 
           "or_scale" = "Original scale", 
            "pec_miss" = "Perc. missings", 
           "dataESS"= "ESS (ref = EB)", 
           "dataEVS"= "EVS", 
           "dataISSP" = "ISSP", 
           "dataLISS"= "LISS", 
           "dataMOT" = "MOT", 
           "nr_waves" = "Nr. of waves"), gof_map = NA) %>%
  kable_classic_2(html_font = "Times", fixed_thead = T, full_width = F) %>%
   add_header_above(c(" " = 1, "Mu" = 8, "Sigma" = 8))%>%
    save_kable("./output/appendix/fixed_meta_regression_table.html")

# Random model for independent variables

model_step_1_att_p <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              method = "ML",
              test = "knha")

## Warning: 1 study with NAs omitted from model fitting.

model_step_1_att_p

## 
## Random-Effects Model (k = 70; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of total heterogeneity): 0.0175 (SE = 0.0030)
## tau (square root of estimated tau^2 value):      0.1322
## I^2 (total heterogeneity / total variability):   99.84%
## H^2 (total variability / sampling variability):  618.98
## 
## Test for Heterogeneity:
## Q(df = 69) = 4378.1393, p-val < .0001
## 
## Model Results:
## 
## estimate      se    tval  df    pval    ci.lb   ci.ub    
##   0.0139  0.0170  0.8150  69  0.4179  -0.0201  0.0478    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_att_p <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat,
              method = "ML",
              test = "knha")

## Warning: 1 study with NAs omitted from model fitting.

summary(model_step_2_att_p)

## 
## Mixed-Effects Model (k = 70; tau^2 estimator: ML)
## 
##   logLik  deviance       AIC       BIC      AICc   
##  37.8247  441.7464  -67.6494  -58.6555  -67.0341   
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0173 (SE = 0.0030)
## tau (square root of estimated tau^2 value):             0.1314
## I^2 (residual heterogeneity / unaccounted variability): 99.83%
## H^2 (unaccounted variability / sampling variability):   592.72
## R^2 (amount of heterogeneity accounted for):            1.23%
## 
## Test for Residual Heterogeneity:
## QE(df = 67) = 3224.2796, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 67) = 0.3409, p-val = 0.7124
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb   ci.ub    
## intrcpt                  0.0208  0.0414   0.5018  67  0.6175  -0.0618  0.1034    
## attitude_cataffective    0.0214  0.0586   0.3654  67  0.7160  -0.0955  0.1384    
## attitude_catcognitive   -0.0161  0.0465  -0.3459  67  0.7305  -0.1089  0.0767    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_att_p <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered,
              method = "ML",
              test = "knha")

## Warning: 1 study with NAs omitted from model fitting.

model_step_3_att_p

## 
## Mixed-Effects Model (k = 70; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0171 (SE = 0.0029)
## tau (square root of estimated tau^2 value):             0.1308
## I^2 (residual heterogeneity / unaccounted variability): 99.83%
## H^2 (unaccounted variability / sampling variability):   579.61
## R^2 (amount of heterogeneity accounted for):            2.11%
## 
## Test for Residual Heterogeneity:
## QE(df = 66) = 3043.5980, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## F(df1 = 3, df2 = 66) = 0.3818, p-val = 0.7664
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb   ci.ub    
## intrcpt                  0.0285  0.0431   0.6621  66  0.5102  -0.0575  0.1145    
## attitude_cataffective    0.0018  0.0654   0.0277  66  0.9780  -0.1288  0.1324    
## attitude_catcognitive   -0.0228  0.0477  -0.4783  66  0.6340  -0.1180  0.0724    
## mean_year_centered       0.0017  0.0024   0.6851  66  0.4957  -0.0032  0.0066    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_4_att_p <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "ML",
              test = "knha")

## Warning: 10 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_att_p

## 
## Mixed-Effects Model (k = 61; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0172 (SE = 0.0032)
## tau (square root of estimated tau^2 value):             0.1313
## I^2 (residual heterogeneity / unaccounted variability): 99.83%
## H^2 (unaccounted variability / sampling variability):   584.24
## R^2 (amount of heterogeneity accounted for):            15.44%
## 
## Test for Residual Heterogeneity:
## QE(df = 47) = 2418.6150, p-val < .0001
## 
## Test of Moderators (coefficients 2:14):
## F(df1 = 13, df2 = 47) = 0.5440, p-val = 0.8843
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb    ci.ub    
## intrcpt                  0.0391  0.1383   0.2829  47  0.7785  -0.2391   0.3173    
## attitude_cataffective   -0.0023  0.0956  -0.0237  47  0.9812  -0.1947   0.1902    
## attitude_catcognitive   -0.0122  0.0591  -0.2065  47  0.8373  -0.1311   0.1067    
## mean_year_centered       0.0000  0.0057   0.0078  47  0.9938  -0.0115   0.0116    
## ambiguousYes            -0.1349  0.0599  -2.2516  47  0.0291  -0.2554  -0.0144  * 
## national                -0.0023  0.1905  -0.0121  47  0.9904  -0.3856   0.3810    
## or_scale                 0.0103  0.0173   0.5975  47  0.5530  -0.0245   0.0452    
## pec_miss                 1.2031  1.4632   0.8223  47  0.4151  -1.7404   4.1466    
## dataESS                 -0.0116  0.1450  -0.0801  47  0.9365  -0.3034   0.2802    
## dataEVS                 -0.0625  0.1785  -0.3505  47  0.7276  -0.4216   0.2965    
## dataISSP                -0.0225  0.0701  -0.3207  47  0.7499  -0.1635   0.1186    
## dataLISS                -0.0065  0.2668  -0.0245  47  0.9806  -0.5433   0.5302    
## dataMOT                 -0.0376  0.1763  -0.2133  47  0.8320  -0.3923   0.3171    
## nr_waves                -0.0205  0.0286  -0.7168  47  0.4770  -0.0781   0.0370    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_1_var_p <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              method = "ML",
              test = "knha")

## Warning: 1 study with NAs omitted from model fitting.

model_step_1_var_p

## 
## Random-Effects Model (k = 70; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of total heterogeneity): 0.0009 (SE = 0.0002)
## tau (square root of estimated tau^2 value):      0.0298
## I^2 (total heterogeneity / total variability):   98.44%
## H^2 (total variability / sampling variability):  63.92
## 
## Test for Heterogeneity:
## Q(df = 69) = 1500.4855, p-val < .0001
## 
## Model Results:
## 
## estimate      se     tval  df    pval    ci.lb    ci.ub     
##  -0.0113  0.0038  -2.9706  69  0.0041  -0.0189  -0.0037  ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_var_p <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat, # cogn as ref cat, sig
              method = "ML",
              test = "knha")

## Warning: 1 study with NAs omitted from model fitting.

model_step_2_var_p

## 
## Mixed-Effects Model (k = 70; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0008 (SE = 0.0002)
## tau (square root of estimated tau^2 value):             0.0280
## I^2 (residual heterogeneity / unaccounted variability): 98.18%
## H^2 (unaccounted variability / sampling variability):   54.87
## R^2 (amount of heterogeneity accounted for):            11.69%
## 
## Test for Residual Heterogeneity:
## QE(df = 67) = 1284.9426, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 67) = 4.0715, p-val = 0.0214
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb    ci.ub     
## intrcpt                  0.0094  0.0087   1.0789  67  0.2845  -0.0080   0.0267     
## attitude_cataffective   -0.0161  0.0125  -1.2839  67  0.2036  -0.0411   0.0089     
## attitude_catcognitive   -0.0273  0.0098  -2.7991  67  0.0067  -0.0468  -0.0078  ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_var_p <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              mods = ~  attitude_cat + mean_year_centered,
              method = "ML",
              test = "knha")

## Warning: 1 study with NAs omitted from model fitting.

model_step_3_var_p

## 
## Mixed-Effects Model (k = 70; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0008 (SE = 0.0001)
## tau (square root of estimated tau^2 value):             0.0279
## I^2 (residual heterogeneity / unaccounted variability): 98.14%
## H^2 (unaccounted variability / sampling variability):   53.63
## R^2 (amount of heterogeneity accounted for):            12.55%
## 
## Test for Residual Heterogeneity:
## QE(df = 66) = 1260.3626, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## F(df1 = 3, df2 = 66) = 2.9285, p-val = 0.0400
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb    ci.ub    
## intrcpt                  0.0072  0.0091   0.7896  66  0.4326  -0.0110   0.0253    
## attitude_cataffective   -0.0109  0.0140  -0.7796  66  0.4384  -0.0389   0.0171    
## attitude_catcognitive   -0.0256  0.0100  -2.5522  66  0.0130  -0.0456  -0.0056  * 
## mean_year_centered      -0.0004  0.0005  -0.8256  66  0.4120  -0.0014   0.0006    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Step 4. Variance
model_step_4_var_p <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              mods = ~  attitude_cat +  mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "ML",
              test = "knha")

## Warning: 10 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_var_p

## 
## Mixed-Effects Model (k = 61; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0005 (SE = 0.0001)
## tau (square root of estimated tau^2 value):             0.0227
## I^2 (residual heterogeneity / unaccounted variability): 97.22%
## H^2 (unaccounted variability / sampling variability):   35.93
## R^2 (amount of heterogeneity accounted for):            32.95%
## 
## Test for Residual Heterogeneity:
## QE(df = 47) = 778.5106, p-val < .0001
## 
## Test of Moderators (coefficients 2:14):
## F(df1 = 13, df2 = 47) = 1.4083, p-val = 0.1913
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb   ci.ub    
## intrcpt                 -0.0199  0.0265  -0.7531  47  0.4552  -0.0731  0.0333    
## attitude_cataffective    0.0023  0.0184   0.1236  47  0.9021  -0.0348  0.0393    
## attitude_catcognitive   -0.0189  0.0106  -1.7840  47  0.0809  -0.0402  0.0024  . 
## mean_year_centered       0.0006  0.0010   0.5890  47  0.5587  -0.0014  0.0025    
## ambiguousYes             0.0106  0.0108   0.9782  47  0.3330  -0.0112  0.0323    
## national                -0.0396  0.0362  -1.0934  47  0.2798  -0.1125  0.0333    
## or_scale                -0.0023  0.0029  -0.7723  47  0.4438  -0.0082  0.0037    
## pec_miss                -0.0416  0.2732  -0.1523  47  0.8796  -0.5912  0.5080    
## dataESS                  0.0097  0.0264   0.3685  47  0.7141  -0.0434  0.0629    
## dataEVS                  0.0149  0.0300   0.4980  47  0.6208  -0.0454  0.0753    
## dataISSP                 0.0289  0.0124   2.3238  47  0.0245   0.0039  0.0539  * 
## dataLISS                 0.0678  0.0487   1.3935  47  0.1700  -0.0301  0.1657    
## dataMOT                  0.0465  0.0340   1.3659  47  0.1785  -0.0220  0.1149    
## nr_waves                 0.0081  0.0053   1.5254  47  0.1339  -0.0026  0.0187    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Fixed model for independent variables

model_step_1_att_fp <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              method = "FE",
              test = "z")

## Warning: 1 study with NAs omitted from model fitting.

model_step_1_att_fp

## 
## Fixed-Effects Model (k = 70)
## 
## I^2 (total heterogeneity / total variability):   98.42%
## H^2 (total variability / sampling variability):  63.45
## 
## Test for Heterogeneity:
## Q(df = 69) = 4378.1393, p-val < .0001
## 
## Model Results:
## 
## estimate      se     zval    pval    ci.lb   ci.ub    
##  -0.0010  0.0006  -1.6083  0.1078  -0.0022  0.0002    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_att_fp <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat,
              method = "FE",
              test = "z")

## Warning: 1 study with NAs omitted from model fitting.

summary(model_step_2_att_fp)

## 
## Fixed-Effects with Moderators Model (k = 70)
## 
##     logLik    deviance         AIC         BIC        AICc   
## -1353.4418   3224.2796   2712.8837   2719.6292   2713.2473   
## 
## I^2 (residual heterogeneity / unaccounted variability): 97.92%
## H^2 (unaccounted variability / sampling variability):   48.12
## R^2 (amount of heterogeneity accounted for):            24.16%
## 
## Test for Residual Heterogeneity:
## QE(df = 67) = 3224.2796, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 1153.8598, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                 -0.0299  0.0011  -27.2526  <.0001  -0.0321  -0.0278  *** 
## attitude_cataffective    0.0855  0.0040   21.2582  <.0001   0.0776   0.0934  *** 
## attitude_catcognitive    0.0411  0.0013   30.5668  <.0001   0.0385   0.0438  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_att_fp <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered,
              method = "FE",
              test = "z")

## Warning: 1 study with NAs omitted from model fitting.

model_step_3_att_fp

## 
## Fixed-Effects with Moderators Model (k = 70)
## 
## I^2 (residual heterogeneity / unaccounted variability): 97.83%
## H^2 (unaccounted variability / sampling variability):   46.12
## R^2 (amount of heterogeneity accounted for):            27.32%
## 
## Test for Residual Heterogeneity:
## QE(df = 66) = 3043.5980, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## QM(df = 3) = 1334.5414, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se     zval    pval    ci.lb    ci.ub      
## intrcpt                 -0.0126  0.0017  -7.4017  <.0001  -0.0159  -0.0092  *** 
## attitude_cataffective    0.0582  0.0045  12.9068  <.0001   0.0493   0.0670  *** 
## attitude_catcognitive    0.0334  0.0015  22.8569  <.0001   0.0306   0.0363  *** 
## mean_year_centered       0.0016  0.0001  13.4418  <.0001   0.0013   0.0018  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_4_att_fp <- rma(yi = mu_time_pred,
              sei = mu_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat + mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "FE",
              test = "z")

## Warning: 10 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_att_fp

## 
## Fixed-Effects with Moderators Model (k = 61)
## 
## I^2 (residual heterogeneity / unaccounted variability): 98.06%
## H^2 (unaccounted variability / sampling variability):   51.46
## R^2 (amount of heterogeneity accounted for):            26.66%
## 
## Test for Residual Heterogeneity:
## QE(df = 47) = 2418.6150, p-val < .0001
## 
## Test of Moderators (coefficients 2:14):
## QM(df = 13) = 1791.2421, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                  0.0112  0.0058    1.9142  0.0556  -0.0003   0.0226    . 
## attitude_cataffective    0.0251  0.0075    3.3311  0.0009   0.0104   0.0399  *** 
## attitude_catcognitive    0.0251  0.0017   14.5202  <.0001   0.0217   0.0285  *** 
## mean_year_centered       0.0008  0.0002    3.6482  0.0003   0.0004   0.0012  *** 
## ambiguousYes            -0.0023  0.0020   -1.1856  0.2358  -0.0062   0.0015      
## national                 0.0130  0.0219    0.5953  0.5517  -0.0299   0.0559      
## or_scale                 0.0113  0.0006   20.1660  <.0001   0.0102   0.0124  *** 
## pec_miss                 0.1015  0.0565    1.7953  0.0726  -0.0093   0.2123    . 
## dataESS                 -0.0248  0.0075   -3.2927  0.0010  -0.0396  -0.0101  *** 
## dataEVS                 -0.0336  0.0039   -8.5223  <.0001  -0.0414  -0.0259  *** 
## dataISSP                -0.0352  0.0027  -12.8527  <.0001  -0.0405  -0.0298  *** 
## dataLISS                -0.0059  0.0248   -0.2370  0.8126  -0.0545   0.0427      
## dataMOT                 -0.0478  0.0220   -2.1709  0.0299  -0.0909  -0.0046    * 
## nr_waves                -0.0176  0.0013  -13.6614  <.0001  -0.0201  -0.0151  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_1_var_fp <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              method = "FE",
              test = "z")

## Warning: 1 study with NAs omitted from model fitting.

model_step_1_var_fp

## 
## Fixed-Effects Model (k = 70)
## 
## I^2 (total heterogeneity / total variability):   95.40%
## H^2 (total variability / sampling variability):  21.75
## 
## Test for Heterogeneity:
## Q(df = 69) = 1500.4855, p-val < .0001
## 
## Model Results:
## 
## estimate      se     zval    pval    ci.lb    ci.ub      
##  -0.0022  0.0004  -4.9300  <.0001  -0.0031  -0.0013  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_var_fp <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              mods = ~ attitude_cat, # cogn as ref cat, sig
              method = "FE",
              test = "z")

## Warning: 1 study with NAs omitted from model fitting.

model_step_2_var_fp

## 
## Fixed-Effects with Moderators Model (k = 70)
## 
## I^2 (residual heterogeneity / unaccounted variability): 94.79%
## H^2 (unaccounted variability / sampling variability):   19.18
## R^2 (amount of heterogeneity accounted for):            11.81%
## 
## Test for Residual Heterogeneity:
## QE(df = 67) = 1284.9426, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 215.5429, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                  0.0071  0.0008    9.1837  <.0001   0.0056   0.0087  *** 
## attitude_cataffective   -0.0180  0.0028   -6.3216  <.0001  -0.0236  -0.0124  *** 
## attitude_catcognitive   -0.0136  0.0010  -14.3265  <.0001  -0.0155  -0.0118  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_var_fp <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              mods = ~  attitude_cat + mean_year_centered,
              method = "FE",
              test = "z")

## Warning: 1 study with NAs omitted from model fitting.

model_step_3_var_fp

## 
## Fixed-Effects with Moderators Model (k = 70)
## 
## I^2 (residual heterogeneity / unaccounted variability): 94.76%
## H^2 (unaccounted variability / sampling variability):   19.10
## R^2 (amount of heterogeneity accounted for):            12.18%
## 
## Test for Residual Heterogeneity:
## QE(df = 66) = 1260.3626, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## QM(df = 3) = 240.1229, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se      zval    pval    ci.lb    ci.ub      
## intrcpt                  0.0026  0.0012    2.1680  0.0302   0.0002   0.0050    * 
## attitude_cataffective   -0.0109  0.0032   -3.4047  0.0007  -0.0171  -0.0046  *** 
## attitude_catcognitive   -0.0116  0.0010  -11.2388  <.0001  -0.0137  -0.0096  *** 
## mean_year_centered      -0.0004  0.0001   -4.9578  <.0001  -0.0006  -0.0002  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Step 4. Variance
model_step_4_var_fp <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = total_reg_results,
              mods = ~  attitude_cat +  mean_year_centered + ambiguous + national + or_scale + pec_miss + data + nr_waves,
              method = "FE",
              test = "z")

## Warning: 10 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_var_fp

## 
## Fixed-Effects with Moderators Model (k = 61)
## 
## I^2 (residual heterogeneity / unaccounted variability): 93.96%
## H^2 (unaccounted variability / sampling variability):   16.56
## R^2 (amount of heterogeneity accounted for):            22.92%
## 
## Test for Residual Heterogeneity:
## QE(df = 47) = 778.5106, p-val < .0001
## 
## Test of Moderators (coefficients 2:14):
## QM(df = 13) = 510.7801, p-val < .0001
## 
## Model Results:
## 
##                        estimate      se     zval    pval    ci.lb    ci.ub      
## intrcpt                 -0.0333  0.0041  -8.0697  <.0001  -0.0414  -0.0252  *** 
## attitude_cataffective   -0.0025  0.0053  -0.4590  0.6462  -0.0129   0.0080      
## attitude_catcognitive   -0.0060  0.0012  -4.9055  <.0001  -0.0084  -0.0036  *** 
## mean_year_centered       0.0004  0.0002   2.5145  0.0119   0.0001   0.0007    * 
## ambiguousYes            -0.0021  0.0014  -1.5153  0.1297  -0.0048   0.0006      
## national                -0.0250  0.0155  -1.6135  0.1066  -0.0553   0.0054      
## or_scale                -0.0021  0.0004  -5.3646  <.0001  -0.0029  -0.0013  *** 
## pec_miss                 0.0262  0.0400   0.6558  0.5120  -0.0521   0.1046      
## dataESS                  0.0200  0.0053   3.7480  0.0002   0.0095   0.0305  *** 
## dataEVS                  0.0226  0.0028   8.1150  <.0001   0.0172   0.0281  *** 
## dataISSP                 0.0303  0.0019  15.6597  <.0001   0.0265   0.0341  *** 
## dataLISS                 0.0626  0.0175   3.5725  0.0004   0.0283   0.0970  *** 
## dataMOT                  0.0369  0.0156   2.3720  0.0177   0.0064   0.0674    * 
## nr_waves                 0.0088  0.0009   9.6467  <.0001   0.0070   0.0106  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Check leaving out the ambiguous variables
ex_amb_df <- subset(total_reg_results, subset = ambiguous == "No")

model_step_1_att_amb <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = ex_amb_df,
              method = "ML",
              test = "knha")

model_step_1_att_amb

## 
## Random-Effects Model (k = 55; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of total heterogeneity): 0.0051 (SE = 0.0010)
## tau (square root of estimated tau^2 value):      0.0714
## I^2 (total heterogeneity / total variability):   99.48%
## H^2 (total variability / sampling variability):  191.17
## 
## Test for Heterogeneity:
## Q(df = 54) = 3468.6471, p-val < .0001
## 
## Model Results:
## 
## estimate      se    tval  df    pval   ci.lb   ci.ub     
##   0.0334  0.0105  3.1898  54  0.0024  0.0124  0.0544  ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_att_amb <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = ex_amb_df,
              mods = ~ attitude_cat,
              method = "ML",
              test = "knha")

model_step_2_att_amb

## 
## Mixed-Effects Model (k = 55; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0048 (SE = 0.0010)
## tau (square root of estimated tau^2 value):             0.0696
## I^2 (residual heterogeneity / unaccounted variability): 99.43%
## H^2 (unaccounted variability / sampling variability):   174.94
## R^2 (amount of heterogeneity accounted for):            4.99%
## 
## Test for Residual Heterogeneity:
## QE(df = 52) = 2479.5930, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 52) = 0.7778, p-val = 0.4647
## 
## Model Results:
## 
##                        estimate      se    tval  df    pval    ci.lb   ci.ub    
## intrcpt                  0.0219  0.0234  0.9348  52  0.3542  -0.0251  0.0689    
## attitude_cataffective    0.0389  0.0342  1.1371  52  0.2607  -0.0297  0.1075    
## attitude_catcognitive    0.0072  0.0269  0.2663  52  0.7911  -0.0469  0.0612    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_att_amb <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = ex_amb_df,
              mods = ~ attitude_cat + mean_year_centered,
              method = "ML",
              test = "knha")

model_step_3_att_amb

## 
## Mixed-Effects Model (k = 55; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0048 (SE = 0.0010)
## tau (square root of estimated tau^2 value):             0.0691
## I^2 (residual heterogeneity / unaccounted variability): 99.41%
## H^2 (unaccounted variability / sampling variability):   169.95
## R^2 (amount of heterogeneity accounted for):            6.29%
## 
## Test for Residual Heterogeneity:
## QE(df = 51) = 2335.9750, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## F(df1 = 3, df2 = 51) = 0.6384, p-val = 0.5938
## 
## Model Results:
## 
##                        estimate      se    tval  df    pval    ci.lb   ci.ub    
## intrcpt                  0.0255  0.0243  1.0482  51  0.2995  -0.0233  0.0743    
## attitude_cataffective    0.0292  0.0379  0.7704  51  0.4446  -0.0469  0.1054    
## attitude_catcognitive    0.0042  0.0275  0.1542  51  0.8780  -0.0510  0.0595    
## mean_year_centered       0.0009  0.0014  0.6086  51  0.5455  -0.0020  0.0038    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_4_att_amb <- rma(yi = mu_time,
              sei = mu_time_sd,
              data = ex_amb_df,
              mods = ~ attitude_cat + mean_year_centered +  national + or_scale + pec_miss + data + nr_waves,
              method = "ML",
              test = "knha")

## Warning: 4 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_att_amb

## 
## Mixed-Effects Model (k = 51; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0031 (SE = 0.0007)
## tau (square root of estimated tau^2 value):             0.0555
## I^2 (residual heterogeneity / unaccounted variability): 99.04%
## H^2 (unaccounted variability / sampling variability):   104.35
## R^2 (amount of heterogeneity accounted for):            43.03%
## 
## Test for Residual Heterogeneity:
## QE(df = 38) = 1755.5124, p-val < .0001
## 
## Test of Moderators (coefficients 2:13):
## F(df1 = 12, df2 = 38) = 2.3579, p-val = 0.0222
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb    ci.ub      
## intrcpt                  0.2462  0.0714   3.4476  38  0.0014   0.1016   0.3908   ** 
## attitude_cataffective   -0.0258  0.0424  -0.6082  38  0.5467  -0.1117   0.0601      
## attitude_catcognitive   -0.0020  0.0267  -0.0766  38  0.9394  -0.0560   0.0520      
## mean_year_centered       0.0041  0.0026   1.5714  38  0.1244  -0.0012   0.0093      
## national                -0.0506  0.0843  -0.6000  38  0.5521  -0.2213   0.1201      
## or_scale                 0.0258  0.0081   3.1776  38  0.0029   0.0094   0.0423   ** 
## pec_miss                -1.6751  0.7572  -2.2122  38  0.0330  -3.2080  -0.1422    * 
## dataESS                 -0.2131  0.0736  -2.8936  38  0.0063  -0.3621  -0.0640   ** 
## dataEVS                 -0.1234  0.0747  -1.6510  38  0.1070  -0.2747   0.0279      
## dataISSP                -0.1226  0.0347  -3.5304  38  0.0011  -0.1928  -0.0523   ** 
## dataLISS                -0.1167  0.1155  -1.0108  38  0.3185  -0.3505   0.1171      
## dataMOT                 -0.1187  0.0778  -1.5261  38  0.1353  -0.2762   0.0388      
## nr_waves                -0.0673  0.0153  -4.3896  38  <.0001  -0.0983  -0.0362  *** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Variance
model_step_1_var_amb <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = ex_amb_df,
              method = "ML",
              test = "knha")

model_step_1_var_amb

## 
## Random-Effects Model (k = 55; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of total heterogeneity): 0.0010 (SE = 0.0002)
## tau (square root of estimated tau^2 value):      0.0323
## I^2 (total heterogeneity / total variability):   98.73%
## H^2 (total variability / sampling variability):  78.92
## 
## Test for Heterogeneity:
## Q(df = 54) = 1366.1029, p-val < .0001
## 
## Model Results:
## 
## estimate      se     tval  df    pval    ci.lb    ci.ub     
##  -0.0142  0.0046  -3.0742  54  0.0033  -0.0234  -0.0049  ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_2_var_amb <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = ex_amb_df,
              mods = ~ attitude_cat, 
              method = "ML",
              test = "knha")

model_step_2_var_amb

## 
## Mixed-Effects Model (k = 55; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0009 (SE = 0.0002)
## tau (square root of estimated tau^2 value):             0.0302
## I^2 (residual heterogeneity / unaccounted variability): 98.49%
## H^2 (unaccounted variability / sampling variability):   66.28
## R^2 (amount of heterogeneity accounted for):            13.00%
## 
## Test for Residual Heterogeneity:
## QE(df = 52) = 1187.1002, p-val < .0001
## 
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 52) = 3.9749, p-val = 0.0248
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb    ci.ub     
## intrcpt                  0.0078  0.0097   0.8048  52  0.4246  -0.0117   0.0273     
## attitude_cataffective   -0.0158  0.0144  -1.1028  52  0.2752  -0.0447   0.0130     
## attitude_catcognitive   -0.0307  0.0112  -2.7471  52  0.0082  -0.0532  -0.0083  ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_3_var_amb <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = ex_amb_df,
              mods = ~  attitude_cat + mean_year_centered,
              method = "ML",
              test = "knha")

model_step_3_var_amb

## 
## Mixed-Effects Model (k = 55; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0009 (SE = 0.0002)
## tau (square root of estimated tau^2 value):             0.0300
## I^2 (residual heterogeneity / unaccounted variability): 98.45%
## H^2 (unaccounted variability / sampling variability):   64.38
## R^2 (amount of heterogeneity accounted for):            14.18%
## 
## Test for Residual Heterogeneity:
## QE(df = 51) = 1172.9749, p-val < .0001
## 
## Test of Moderators (coefficients 2:4):
## F(df1 = 3, df2 = 51) = 2.8334, p-val = 0.0473
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb    ci.ub    
## intrcpt                  0.0057  0.0101   0.5630  51  0.5759  -0.0146   0.0260    
## attitude_cataffective   -0.0105  0.0159  -0.6589  51  0.5129  -0.0425   0.0215    
## attitude_catcognitive   -0.0291  0.0114  -2.5510  51  0.0138  -0.0520  -0.0062  * 
## mean_year_centered      -0.0005  0.0006  -0.7859  51  0.4356  -0.0017   0.0007    
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

model_step_4_var_amb <- rma(yi = sig_time_pred,
              sei = sig_time_sd_pred,
              data = ex_amb_df,
              mods = ~  attitude_cat +  mean_year_centered + national + or_scale + pec_miss + data + nr_waves,
              method = "ML",
              test = "knha")

## Warning: 4 studies with NAs omitted from model fitting.

## Warning: Redundant predictors dropped from the model.

model_step_4_var_amb

## 
## Mixed-Effects Model (k = 51; tau^2 estimator: ML)
## 
## tau^2 (estimated amount of residual heterogeneity):     0.0006 (SE = 0.0001)
## tau (square root of estimated tau^2 value):             0.0235
## I^2 (residual heterogeneity / unaccounted variability): 97.34%
## H^2 (unaccounted variability / sampling variability):   37.64
## R^2 (amount of heterogeneity accounted for):            35.29%
## 
## Test for Residual Heterogeneity:
## QE(df = 38) = 741.4092, p-val < .0001
## 
## Test of Moderators (coefficients 2:13):
## F(df1 = 12, df2 = 38) = 1.4856, p-val = 0.1724
## 
## Model Results:
## 
##                        estimate      se     tval  df    pval    ci.lb   ci.ub     
## intrcpt                 -0.0505  0.0327  -1.5408  38  0.1316  -0.1167  0.0158     
## attitude_cataffective    0.0061  0.0194   0.3168  38  0.7531  -0.0331  0.0454     
## attitude_catcognitive   -0.0183  0.0117  -1.5673  38  0.1253  -0.0419  0.0053     
## mean_year_centered       0.0000  0.0011   0.0020  38  0.9984  -0.0022  0.0022     
## national                -0.0283  0.0382  -0.7417  38  0.4628  -0.1057  0.0490     
## or_scale                -0.0042  0.0035  -1.2112  38  0.2333  -0.0113  0.0028     
## pec_miss                 0.2469  0.3417   0.7225  38  0.4744  -0.4448  0.9386     
## dataESS                  0.0379  0.0330   1.1473  38  0.2584  -0.0290  0.1047     
## dataEVS                  0.0237  0.0316   0.7493  38  0.4583  -0.0403  0.0876     
## dataISSP                 0.0416  0.0152   2.7430  38  0.0092   0.0109  0.0723  ** 
## dataLISS                 0.0792  0.0512   1.5465  38  0.1303  -0.0245  0.1828     
## dataMOT                  0.0560  0.0355   1.5765  38  0.1232  -0.0159  0.1279     
## nr_waves                 0.0150  0.0069   2.1611  38  0.0371   0.0009  0.0290   * 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# Still need to adjust, the fit statitics are not complete

models <- list("M1" = model_step_1_att_amb, "M2" = model_step_2_att_amb, "M3" = model_step_3_att_amb, "M4" = model_step_4_att_amb, "M1" =  model_step_1_var_amb, "M2"= model_step_2_var_amb, "M3"=  model_step_3_var_amb, "M4" = model_step_4_var_amb)


modelsummary(models, output = "kableExtra", statistic = 'std.error', stars = TRUE, shape = term ~ model + statistic, title = "Appendix X. Meta-regression excluding ambiguous variables.", fmt = fmt_statistic(estimate = 3, std.error =3), coef_rename = c("overall" ="Intercept", 
           "intercept" = "Intercept", 
           "attitude_cataffective" = "Affective attitude (ref = beh)",
           "attitude_catcognitive" = "Cognitive attitude", 
           "mean_year_centered" =   "Mean year centered", 
           "national" = "National (ref = no)", 
           "or_scale" = "Original scale", 
            "pec_miss" = "Perc. missings", 
           "dataESS"= "ESS (ref = EB)", 
           "dataEVS"= "EVS", 
           "dataISSP" = "ISSP", 
           "dataLISS"= "LISS", 
           "dataMOT" = "MOT", 
           "nr_waves" = "Nr. of waves"), gof_map = NA) %>%
  kable_classic_2(html_font = "Times", fixed_thead = T, full_width = F) %>%
   add_header_above(c(" " = 1, "Mu" = 8, "Sigma" = 8))%>%
    save_kable("./output/appendix/meta_regression_table_excluding_ambiguous.html")