---
title: "Appendix 4"
output: pdf_document
---

```{r setup, include=FALSE, message=FALSE, warning=FALSE}
knitr::opts_chunk$set(echo = TRUE)
library(tidyverse)
mathia <- read.csv(file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/data/MATHia_Course_2_(Grade-7)-All_Data_2019-2020/course2_1920_sample1_500students_datashop.txt", header = TRUE, sep = "\t")
imp_workspaces <- read_csv("/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/data/TOC-msms_course2.txt")
```

### In this technical appendix, we work with the MATHia course 2 (Grade 7) dataset of 500 randomly selected students. Using Gaussian Graphical Models, we find out strongly related workspaces and the partial correlation coefficient for each pair. We later using retrospective study design to evaluate the prerequisite relation between the workspaces.

```{r message=FALSE, warning=FALSE}
head(mathia)
head(imp_workspaces)
```

### Getting workspaces from MATHia 'shipped' Course 2 curriculum.

```{r message=FALSE, warning=FALSE}
imp_workspaces2 <- imp_workspaces %>%
  filter(substr(`TOC for SequenceTemplate : Middle School Math Solution Course 2 ( msms_course2 )`, 
                 1, 1) == "S") %>%
  mutate(Start = str_locate(`TOC for SequenceTemplate : Middle School Math Solution Course 2 ( msms_course2 )`,
                            "\\("),
         End = nchar(`TOC for SequenceTemplate : Middle School Math Solution Course 2 ( msms_course2 )`)) %>%
  mutate(wks = substr(`TOC for SequenceTemplate : Middle School Math Solution Course 2 ( msms_course2 )`,
                      Start+1, End-1))
imp_workspaces3 <- imp_workspaces2 %>%
  select(wks) %>%
  mutate(IF = str_locate(wks, "\\(")) 
imp_workspaces3$IF <- ifelse(is.na(imp_workspaces3$IF)==TRUE, 0,
                             imp_workspaces3$IF)
imp_workspaces4 <- imp_workspaces3 %>%
  mutate(workspaces = substr(wks, IF+1, length(wks))) %>%
  select(workspaces) %>%
  filter(!(workspaces %in% c("pre_launch_protocol", "factoring_expression", 
                             "scale_drawings_", "understanding_opposite"))) %>%
  add_row(workspaces = c("factoring_expressions", "scale_drawings_3", 
                         "understanding_opposites"))
```

### The workspaces from MATHia Course 2 'shipped' curriculum.

```{r message=FALSE, warning=FALSE}
sort(imp_workspaces4$workspaces)
```

### Filtering workspaces from the ones in MATHia Course 2 'shipped' curriculum.

```{r message=FALSE, warning=FALSE}
mathia2 <- mathia %>%
  filter(Level..Workspace.Id. %in% imp_workspaces4$workspaces)
```

```{r message=FALSE, warning=FALSE}
# Sanity check
imp_workspaces4$workspaces[!imp_workspaces4$workspaces %in% 
                             unique(mathia$Level..Workspace.Id.)]
length(unique(mathia2$Level..Workspace.Id.))
```

### We want to find success rate for first attempts of all steps in a workspace. So filtering by `Attempt.At.Step == 1`. 

```{r message=FALSE, warning=FALSE}
mathia3 <- mathia2 %>%
  group_by(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name) %>%
  filter(Attempt.At.Step == 1) %>%
  select(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name, 
         Outcome, Attempt.At.Step)
```

### Converting `Outcome` to `success`, its numeric version

```{r message=FALSE, warning=FALSE}
mathia4 <- mathia3 %>%
  mutate(success = ifelse(Outcome == "OK", 1, 0)) %>%
  select(-Attempt.At.Step)
```

### Calculating mean success rate as `success_rate`.

```{r message=FALSE, warning=FALSE}
mathia5 <- mathia4 %>%
  ungroup() %>%
  group_by(Anon.Student.Id, Level..Workspace.Id.) %>%
  summarise(success_rate = mean(success))
```

### Transforming the data frame using `pivot_wider` to be able to compute correlations.

```{r message=FALSE, warning=FALSE}
mathia6 <- mathia5 %>%
  ungroup() %>%
  pivot_wider(names_from = Level..Workspace.Id., 
              values_from = success_rate) %>%
  select(-Anon.Student.Id)
```

### Applying logit function.
 
```{r message=FALSE, warning=FALSE}
mathia7 <- mathia6
logit <- function(v){
  if (is.na(v)==TRUE){
    return(NA)
  }
  if (is.nan(v) == TRUE){
    return(NA)
  }
  if (v==1){
    v = 0.9999
  }
  if (v==0){
    v = 0.0001
  }
  return(log(v/(1-v)))
}
for (i in colnames(mathia7)){
  mathia7[[i]] <- lapply(mathia7[[i]], logit)
}
for (i in colnames(mathia7)){
  mathia7[[i]] <- as.numeric(mathia7[[i]])
}
```

### Validating the normality assumption for some workspaces.

```{r message=FALSE, warning=FALSE}
hist(mathia7[[1]])
hist(mathia7[[15]])
hist(mathia7[[21]])
hist(mathia7[[41]])
hist(mathia7[[73]])
```

### Computing pairwise correlations and converting the matrix to positive definite.

```{r message=FALSE, warning=FALSE}
# mathia_cor_pd <- psych::corFiml(mathia7) gives error
mathia_cor_success_rate_pairwise <- cor(mathia7, use = "pairwise.complete.obs")
mathia_cor_success_rate <- psych::cor.smooth(mathia_cor_success_rate_pairwise)
```

### GGM plot where student performance metric is success rate of first attempts for all steps in a  workspace.

```{r message=FALSE, warning=FALSE}
ggm_success_rate <- qgraph::qgraph(
      mathia_cor_success_rate,
      layout = "spring",
      graph = "glasso",
      labels = TRUE,
      minimum = 0.1,
      sampleSize = 500
    )
```

### Getting the pairs of related workspaces from GGM along with partial correlation coefficients.

```{r message=FALSE, warning=FALSE}
workspaces_data_success_rate <- data.frame(Node = 1 : 76,
                              Workspace = colnames(mathia_cor_success_rate))
```

```{r message=FALSE, warning=FALSE}
df_success_rate <- data.frame(Node_1 = ggm_success_rate$Edgelist$from,
                              Node_2 = ggm_success_rate$Edgelist$to,
                              Weight = ggm_success_rate$Edgelist$weight) %>%
  arrange(desc(Weight)) %>%
  full_join(workspaces_data_success_rate, by = c("Node_1" = "Node")) %>%
  rename(Workspace_1 = Workspace) %>%
  full_join(workspaces_data_success_rate, by = c("Node_2" = "Node")) %>%
  rename(Workspace_2 = Workspace) %>%
  relocate(Node_1, Workspace_1, Node_2, Workspace_2, Weight)
```

```{r message=FALSE, warning=FALSE}
head(df_success_rate)
```

### Now, the student performance metric is assistance score over a workspace. Calculating assistance score for a student over a workspace.

```{r message=FALSE, warning=FALSE}
mathia8 <- mathia2 %>%
  group_by(Anon.Student.Id, Level..Workspace.Id.) %>%
  select(Anon.Student.Id, Level..Workspace.Id., Outcome)
```

```{r message=FALSE, warning=FALSE}
mathia9 <- mathia8 %>%
  mutate(assistance = ifelse(Outcome == "OK", 0, 1)) %>%
  select(-Outcome) 
```

```{r message=FALSE, warning=FALSE}
mathia10 <- mathia9 %>%
  summarise(assistance_score = sum(assistance))
```

### Transforming the data frame using `pivot_wider` to be able to compute correlations.

```{r message=FALSE, warning=FALSE}
mathia11 <- mathia10 %>%
  ungroup() %>%
  pivot_wider(names_from = Level..Workspace.Id., 
              values_from = assistance_score) %>%
  select(-Anon.Student.Id)
```

```{r message=FALSE, warning=FALSE}
mathia12 <- mathia11
log2 <- function(num){
  if (is.na(num)==TRUE || is.null(num)==TRUE){
    return(NA)
  }
  if (num == 0){
    return(log(0.0001))
  }
  return(log(num))
}
for (i in colnames(mathia12)){
  mathia12[[i]] <- lapply(mathia12[[i]], log2)
}
for (i in colnames(mathia12)){
  mathia12[[i]] <- as.numeric(mathia12[[i]])
}
```

### Validating the normality assumption for some workspaces.

```{r message=FALSE, warning=FALSE}
hist(mathia12[[11]])
hist(mathia12[[5]])
hist(mathia12[[31]])
hist(mathia12[[55]])
hist(mathia12[[75]])
```

### Computing pairwise correlations and converting the matrix to positive definite.

```{r message=FALSE, warning=FALSE}
# mathia_cor_pd <- psych::corFiml(mathia12) gives error
mathia_cor_assistance_score_pairwise <- cor(mathia12, 
                                            use = "pairwise.complete.obs")
mathia_cor_assistance_score <- 
  psych::cor.smooth(mathia_cor_assistance_score_pairwise)
```

### GGM plot where student performance metric is assistance score across a  workspace.

```{r message=FALSE, warning=FALSE}
ggm_assistance_score <- qgraph::qgraph(
      mathia_cor_assistance_score,
      layout = "spring",
      graph = "glasso",
      labels = TRUE,
      minimum = 0.1,
      sampleSize = 500
    )
```

### Getting the pairs of related workspaces from GGM along with partial correlation coefficients.

```{r message=FALSE, warning=FALSE}
workspaces_data_assistance_score <- data.frame(Node = 1 : 76,
                              Workspace = colnames(mathia_cor_assistance_score))
```

```{r message=FALSE, warning=FALSE}
df_assistance_score <- data.frame(Node_1 = ggm_assistance_score$Edgelist$from,
                              Node_2 = ggm_assistance_score$Edgelist$to,
                              Weight = ggm_assistance_score$Edgelist$weight) %>%
  arrange(desc(Weight)) %>%
  full_join(workspaces_data_assistance_score, by = c("Node_1" = "Node")) %>%
  rename(Workspace_1 = Workspace) %>%
  full_join(workspaces_data_assistance_score, by = c("Node_2" = "Node")) %>%
  rename(Workspace_2 = Workspace) %>%
  relocate(Node_1, Workspace_1, Node_2, Workspace_2, Weight)
```

```{r message=FALSE, warning=FALSE}
head(df_assistance_score)
```


```{r include=FALSE}
#write_csv(df_success_rate, file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/data/df_success_rate.csv")
```

```{r include=FALSE} 
#write_csv(df_assistance_score, file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/data/df_assistance_score.csv")
```

```{r include=FALSE}
#save.image("/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/code/MATHIA_FINAL.RData")
```

```{r include=FALSE}
#load("/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/code/MATHIA_FINAL.RData")
```

### Examining the GGM plot when student performance metric is 'first 2 attempts' for all steps and 'first 3 attempts' for all steps in a workspace.

```{r message=FALSE, warning=FALSE}
mathia13 <- mathia2 %>%
  group_by(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name) %>%
  filter(Attempt.At.Step %in% c(1, 2)) %>%
  select(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name, 
         Outcome, Attempt.At.Step) %>%
  mutate(success = ifelse(Outcome == "OK", 1, 0)) %>%
  select(-Outcome) %>%
  ungroup() %>%
  group_by(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name) %>%
  summarise(success_rate1 = mean(success)) %>%
  ungroup() %>%
  group_by(Anon.Student.Id, Level..Workspace.Id.) %>%
  summarise(success_rate = mean(success_rate1)) %>%
  pivot_wider(names_from = Level..Workspace.Id., values_from = success_rate) %>%
  ungroup() %>%
  select(-Anon.Student.Id)
```

```{r message=FALSE, warning=FALSE}
for (i in colnames(mathia13)){
  mathia13[[i]] <- lapply(mathia13[[i]], logit)
}
for (i in colnames(mathia13)){
  mathia13[[i]] <- as.numeric(mathia13[[i]])
}
mathia_cor_success_rate_pairwise_2attempts <- cor(mathia13, 
                                                  use = "pairwise.complete.obs")
mathia_cor_success_rate_2attempts <- 
  psych::cor.smooth(mathia_cor_success_rate_pairwise_2attempts)
ggm_success_rate_2attempts <- qgraph::qgraph(
      mathia_cor_success_rate_2attempts,
      layout = "spring",
      graph = "glasso",
      labels = TRUE,
      minimum = 0.1,
      sampleSize = 500
    )
```

```{r include=FALSE}
workspaces_data_success_rate_2attempts <- data.frame(Node = 1 : 76,
                              Workspace = colnames(mathia_cor_success_rate_2attempts))
df_success_rate_2attempts <- 
  data.frame(Node_1 = ggm_success_rate_2attempts$Edgelist$from,
                              Node_2 = ggm_success_rate_2attempts$Edgelist$to,
                              
             Weight = ggm_success_rate_2attempts$Edgelist$weight) %>%
  arrange(desc(Weight)) %>%
  full_join(workspaces_data_success_rate_2attempts, 
            by = c("Node_1" = "Node")) %>%
  rename(Workspace_1 = Workspace) %>%
  full_join(workspaces_data_success_rate_2attempts, 
            by = c("Node_2" = "Node")) %>%
  rename(Workspace_2 = Workspace) %>%
  relocate(Node_1, Workspace_1, Node_2, Workspace_2, Weight)
```

```{r message=FALSE, warning=FALSE}
mathia14 <- mathia2 %>%
  group_by(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name) %>%
  filter(Attempt.At.Step %in% c(1, 2, 3)) %>%
  select(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name,
         Outcome, Attempt.At.Step) %>%
  mutate(success = ifelse(Outcome == "OK", 1, 0)) %>%
  select(-Outcome) %>%
  ungroup() %>%
  group_by(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name) %>%
  summarise(success_rate1 = mean(success)) %>%
  ungroup() %>%
  group_by(Anon.Student.Id, Level..Workspace.Id.) %>%
  summarise(success_rate = mean(success_rate1)) %>%
  pivot_wider(names_from = Level..Workspace.Id., values_from = success_rate) %>%
  ungroup() %>%
  select(-Anon.Student.Id)
```

```{r message=FALSE, warning=FALSE}
for (i in colnames(mathia14)){
  mathia14[[i]] <- lapply(mathia14[[i]], logit)
}
for (i in colnames(mathia14)){
  mathia14[[i]] <- as.numeric(mathia14[[i]])
}
mathia_cor_success_rate_pairwise_3attempts <- cor(mathia14, 
                                                  use = "pairwise.complete.obs")
mathia_cor_success_rate_3attempts <- 
  psych::cor.smooth(mathia_cor_success_rate_pairwise_3attempts)
ggm_success_rate_3attempts <- qgraph::qgraph(
      mathia_cor_success_rate_3attempts,
      layout = "spring",
      graph = "glasso",
      labels = TRUE,
      minimum = 0.1,
      sampleSize = 500
    )
```

```{r include=FALSE}
workspaces_data_success_rate_3attempts <- data.frame(Node = 1 : 76,
                              Workspace = 
                                colnames(mathia_cor_success_rate_3attempts))
df_success_rate_3attempts <- data.frame(Node_1 = 
                                          ggm_success_rate_3attempts$Edgelist$from,
                              Node_2 = 
                                ggm_success_rate_3attempts$Edgelist$to,
                              Weight = 
                                ggm_success_rate_3attempts$Edgelist$weight) %>%
  arrange(desc(Weight)) %>%
  full_join(workspaces_data_success_rate_3attempts, 
            by = c("Node_1" = "Node")) %>%
  rename(Workspace_1 = Workspace) %>%
  full_join(workspaces_data_success_rate_3attempts, 
            by = c("Node_2" = "Node")) %>%
  rename(Workspace_2 = Workspace) %>%
  relocate(Node_1, Workspace_1, Node_2, Workspace_2, Weight)
```

```{r include=FALSE}
mathia15 <- mathia2 %>%
  group_by(Anon.Student.Id, Level..Workspace.Id., Problem.Name, Step.Name) %>%
  summarise(total = length(Outcome))
```

```{r include=FALSE}
summary(mathia15$total)
levels(as.factor(mathia15$total))
summary(as.factor(mathia15$total))
```

```{r include=FALSE}
#write_csv(df_success_rate_2attempts, file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/data/df_success_rate_2attempts.csv")
#write_csv(df_success_rate_3attempts, file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/data/df_success_rate_3attempts.csv")
#write_csv(mathia2, file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/data/mathia_filtered.csv")
```

```{r include=FALSE}
#load("/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/code/MATHIA_FINAL.RData")
```

```{r include=FALSE}
#save(ggm_success_rate, file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/code/success_rate_ggm_plot.RData")
#save(ggm_assistance_score, file = "/Users/smeetpoladia/Desktop/Carnegie Mellon University/Department of Statistics and Data Science/Spring_2021/36-726/HCI_Prerequisite_Relations/data_code/code/assistance_score_ggm_plot.RData")
```

























