---
title: ""
author: ""
output: html_document
---

```{r setup}
library(tidyverse)
library(tidytext)
```

# PISA data

```{r read-pisa}
pisa_au <- read_csv("data/pisa_au.csv")

library(labelled)
pisa_au_science <- pisa_au %>% 
  filter(science_fun < 5) %>%
  filter(!is.na(science_time)) %>% 
  select(science, 
         science_fun, 
         science_time, 
         stuweight) %>%
  mutate(science_fun = to_factor(science_fun, 
                                 ordered = TRUE, 
                                 drop_unused_labels = TRUE)) %>%
  mutate(science_time = as.numeric(science_time)) %>%
  filter(science_time > 0)
  
```

```{r}
ggplot(pisa_au_science, 
       aes(x = science_time, 
           y = science,
           colour = science_fun)) + 
  geom_point(alpha = 0.1) + 
  scale_colour_brewer("Enjoy science", 
                      palette = "Dark2") +
  facet_wrap(~science_fun, 
             ncol = 2) + 
  scale_x_log10() +
  # geom_smooth(method="lm", se=FALSE) + 
  theme(legend.position = "bottom") + 
  xlab("Time spent studying science per week (mins") + 
  ylab("Synthetic science score")
```

```{r}
pisa_au_science_log10 <- pisa_au_science %>%
  mutate(log_science_time = log10(science_time)) %>%
  mutate(science_fun_c = factor(science_fun, ordered = FALSE))

mod1 <- lm(science ~ log_science_time + science_fun_c, 
           data = pisa_au_science, 
           weights = stuweight)

mod2 <- lm(science ~ log_science_time * science_fun_c, 
           data = pisa_au_science, 
           weights = stuweight)
```


# five minute exercise

- Write out the equations for both models. (Ignore the log transformation.)
- Make a **hand** sketch of both models.

# Using the PISA data: How does science score relate to text anxiety and gender?

- Make a plot of science by anxtest, coloured by gender. Does it look like an interaction term might be necessary?

```{r }
# filter out the missings in science_time, and only look at science, gender, anxtext, and stuweight
pisa_au_science <- pisa_au %>% 
  filter(!is.na(science_time)) %>%
  select(science, gender, anxtest, stuweight) 

# plot of science by anxtest, coloured by gender. Does it look like an interaction term might be necessary?
ggplot(pisa_au_science, 
       aes(x = , 
           y = , 
           colour = )) + 
  geom_smooth(method = "lm")
```

- Fit the model with `science` score as the response and `gender` and `anxtest`. 

```{r}
sci_lm1 <- lm(science ~  +  ,
              data = pisa_au_science, 
              weights=stuweight)
tidy(???)
glance(???)
```

- Try an interaction between gender and anxtest. 

```{r}
sci_lm2 <- lm(science ~ ??? * ???, data=pisa_au_science, weights=stuweight)
```


- Which is the better model?

# final model exercise

Build the best model you can for science scoreby exploring  these variables: 

- math score
- reading score
- tvs
- books
- breakfast

Feel free to choose others. (Code provided in exercise is just a sample, and needs to be modified.)

```{r echo=FALSE, eval=FALSE}

pisa_au_science <- pisa_au %>% 
  filter(science_fun < 5) %>%
  filter(!is.na(science_time)) %>%
  filter(!is.na(anxtest)) %>%
  select(science, math, read, science_fun, science_time, breakfast, gender, anxtest, books, tvs, stuweight) %>%
  mutate(science_time = as.numeric(science_time)) %>%
  filter(science_time>0) %>%
  mutate(log_science_time = log10(science_time)) %>%
  mutate(science_fun_c = factor(science_fun, ordered=FALSE))
sci_lm <- lm(science ~ math + read + science_fun_c + science_time + gender*anxtest, data=pisa_au_science, weights=stuweight)
tidy(sci_lm)
glance(sci_lm)
```

# back to tidy text


```{r get-sentiment-afinn}
get_sentiments("afinn")
```

# Exploring sentiment in Jane Austen

- The `janeaustenr` package contains the full texts, ready for analysis for for Jane Austen's 6 completed novels: 

1. "Sense and Sensibility"
2. "Pride and Prejudice"
3. "Mansfield Park"
4. "Emma"
5. "Northanger Abbey"
6. "Persuasion"


# Exploring sentiment in Jane Austen

```{r show-jane-austen}
library(janeaustenr)
library(stringr)

tidy_books <- austen_books() %>%
  group_by(book) %>%
  mutate(linenumber = row_number(),
         chapter = cumsum(str_detect(text, 
                                     regex("^chapter [\\divxlc]", 
                                           ignore_case = TRUE)))) %>%
  ungroup() %>%
  unnest_tokens(output = word, 
                input = text)
```

# Count joyful words in "Emma"

```{r count-joy}
nrc_joy <- get_sentiments("nrc") %>% 
  filter(sentiment == "joy")

tidy_books %>%
  filter(book == "Emma") %>%
  inner_join(nrc_joy) %>%
  count(word, sort = TRUE)
```

# Exercise 1: 

- What are the most common "anger" words used in Emma?
- What are the most common "surprise" words used in Emma?

```{r show-anger-surprise}
nrc_anger <- get_sentiments("nrc") %>% 
  filter(sentiment == "anger")

nrc_surprise <- get_sentiments("nrc") %>% 
  filter(sentiment == "surprise")

tidy_books %>%
  filter(book == "Emma") %>%
  inner_join(nrc_anger) %>%
  count(word, sort = TRUE)

tidy_books %>%
  filter(book == "Emma") %>%
  inner_join(nrc_surprise) %>%
  count(word, sort = TRUE)
```

Comparing lexicons

```{r compare-lexi}
emma_nrc 

emma_bing

emma_nrc %>% 
  count(sentiment) %>% 
  mutate(n / sum(n))

emma_bing %>% 
  count(sentiment) %>% 
  mutate(n / sum(n))
```

```{r compare-lexi-pos-neg}
emma_afinn %>% 
  mutate(sentiment = ifelse(value > 0, 
                            "positive", 
                            "negative")) %>% 
  count(sentiment) %>% 
  mutate(n / sum(n))
```

# Your Turn: Exercise 2

- Using your choice of lexicon (nrc, bing, or afinn) compute the proportion of positive words in each of Austen's books.

```{r}
nrc <- get_sentiments("nrc")
bing <- get_sentiments("bing")

tidy_books_sent <- tidy_books %>%
  inner_join(bing) %>%
  group_by(book) %>%
  count(sentiment, sort = TRUE) %>%
  mutate(prop = n / sum(n))

tidy_books_sent
```

- Which book is the most positive? negative?

```{r arrange}
tidy_books_sent %>% 
  arrange(-n)
```


# Your turn: Exercise 3

```{r read-simpsons}
scripts <- read_csv("data/simpsons_script_lines.csv")
chs <- read_csv("data/simpsons_characters.csv")
sc <- left_join(scripts, chs, by = c("character_id" = "id"))

```

```{r create-words}
sc_word <- sc %>%
  unnest_tokens(output = word, 
                input = spoken_words) %>%
  anti_join(stop_words) %>%
  count(name, word) %>%
  filter(!is.na(word))

sc_s <- sc_word %>% 
  inner_join(get_sentiments("afinn"), by = "word")

sc_s
```


1. Bart Simpson is featured at various ages. How has the sentiment of his words changed over his life?

```{r}
# explore barts
sc %>% count(name, sort = TRUE) %>%
  filter(grepl("Bart", name)) %>% print(n = 50)

bart_names <- c("Bart Simpson", "Baby Bart", 
                "1-Year-Old Bart", "2-Year-Old Bart", 
                "5-Year-Old Bart", "80-Year-Old Bart")
bart <- sc %>% filter(name %in% bart_names)



sc_bart <- sc %>%
  filter(str_detect(name, "Bart")) %>%
  # find non exact matches for "bart simpson"
  filter(name != "Bart Simpson") %>%
  unnest_tokens(output = word,
                input = spoken_words) %>%
  anti_join(stop_words) 

sc_bart
```


2. Repeat the sentiment analysis with the NRC lexicon. What character is the most "angry"? "joyful"?

```{r src-word}
sc_word %>%
  inner_join(nrc) %>%
  count(name, sentiment) %>%
  arrange(-n)
```



# Your Turn: Sentiment analysis

```{r}
load("data/afl_twitter.rda")
afl
```

We need to break the text of each tweet into words, tag the words with sentiments, and make a cumulative score for each tweet.

- Which tweeter is the most positive? negative?
- Is there a day that spirits were higher in the tweets? Or when tweets were more negative?
- Does the tweeter `aflratings` have a trend in positivity or negativity?

```{r eval=FALSE, echo=FALSE}
afl_sentiment <- afl %>% 
  # group by something
  group_by() %>%
  # unnest the tokens
  unnest_tokens() %>%
  inner_join(get_sentiments("afinn")) %>%
  summarise(sentiment = mean(score, na.rm=T)) 

afl_sentiment <- afl %>% 
  select(status_id, 
         screen_name, 
         created_at, 
         text) %>% 
  left_join(afl_sentiment, by=c(???))

afl_sentiment %>% 
  group_by(???) %>%
  ???(s = mean(sentiment, na.rm=TRUE)) %>% arrange(desc(s))
afl_sentiment %>% mutate(day = ???)) %>% ggplot(aes(x=day, y=sentiment)) + geom_point() + geom_smooth(se=FALSE)
afl_sentiment %>% filter(screen_name == ???) %>% 
  mutate(day = ???) %>%
  ggplot(aes(x=day, y=sentiment)) + geom_point() + geom_smooth()
```


# Your turn

- When was the final played last year?
- What is the range of dates of this data?
- Who is the most frequent tweeter using this hashtag?
- Are there some days that have more tweets than others?
- Are there some hours of the day that are more common tweet times?


# Sentiment analysis

We need to break the text of each tweet into words, tag the words with sentiments, and make a cumulative score for each tweet.

