hw5p1-wiki-vandalism.Rmd

---
title: "Detecting Vandalism on Wikipedia"
author: "Terrel Shumway"
date: "05/02/2015"
output:
  html_document:
    theme: readable
---

This document presents answers for homework 5 part 1.

## Getting and Cleaning the Data
```{r}
baseurl = "https://courses.edx.org/c4x/MITx/15.071x_2/asset/"

getdata = function(local){
  if(!file.exists(local)){
    library(downloader)
    remote = paste0(baseurl,local)
    print(remote)
    download(remote,local)
  }
  read.csv(local,stringsAsFactors=FALSE)
}

data = getdata("wiki.csv")
data$Vandal = factor(data$Vandal)
```

## Bag of Words
problem 1.1:
```{r}
sum(data$Vandal==1)
```

```{r}
library(tm)
library(SnowballC)
added = tm_map(Corpus(VectorSource(data$Added)),function(x){
  stemDocument(removeWords(x,stopwords("english")))
})
dtmA = DocumentTermMatrix(added)
wordsAdded = rowSums(as.matrix(dtmA))
```
problem 1.2:
```{r}
ncol(dtmA)
```
problem 1.3:
```{r}
dtmA = as.matrix(removeSparseTerms(dtmA,0.997))
ncol(dtmA)
```

problem 1.4:
```{r}
corpus = tm_map(Corpus(VectorSource(data$Removed)),function(x){
  stemDocument(removeWords(x,stopwords("english")))
})
dtmR = DocumentTermMatrix(corpus)
wordsRemoved = rowSums(as.matrix(dtmR))
dtmR = as.matrix(removeSparseTerms(dtmR,0.997))
ncol(dtmR)
```


```{r}
colnames(dtmA) = paste0("A",colnames(dtmA))
colnames(dtmR) = paste0("R",colnames(dtmR))
ww = as.data.frame(cbind(dtmA,dtmR))
ww$Vandal = data$Vandal
```

problem 1.5:
```{r}
library(caTools)
set.seed(123)
spl = sample.split(ww$Vandal,SplitRatio=.7)
train = ww[spl,]
test = ww[!spl,]

bl.acc = sum(test$Vandal==0)/nrow(test)
bl.acc
```

problem 1.6:
```{r}
library(rpart)
library(rpart.plot)
m1 = rpart(Vandal~.,data=train)
p1 = predict(m1,newdata=test,type="class")
conf = table(test$Vandal,p1)
acc = sum(diag(conf))/nrow(test)
acc
```

problem 1.7:
```{r}
prp(m1)
```



## Problem-Specific Knowledge

```{r}
ww2 = ww
ww2$HTTP = ifelse(grepl("http",data$Added,fixed=TRUE),0,1)
train2 = ww2[spl,]
test2 = ww2[!spl,]

m2 = rpart(Vandal~.,data=train2)
p2 = predict(m2,newdata=test2,type="class")
conf = table(test2$Vandal,p2)
acc = sum(diag(conf))/nrow(test2)
acc

```

problem 2.3: `r mean(wordsAdded)`
```{r}

ww2$NumWordsAdded = wordsAdded
ww2$NumWordsRemoved = wordsRemoved

train3 = ww2[spl,]
test3 = ww2[!spl,]

m3 = rpart(Vandal~.,data=train3)
p3 = predict(m3,newdata=test3,type="class")
conf = table(test3$Vandal,p3)
acc = sum(diag(conf))/nrow(test3)

```
problem 2.4: $accuracy = `r acc`$



```{r}

ww2$Minor = data$Minor
ww2$Loggedin = data$Loggedin

train4 = ww2[spl,]
test4 = ww2[!spl,]

m4 = rpart(Vandal~.,data=train4,method="class")
p4 = predict(m4,newdata=test4,type="class")
conf = table(test4$Vandal,p4)
acc = sum(diag(conf))/nrow(test4)

```
problem 3.1: $accuracy = `r acc`$

problem 3.2:
```{r,echo=FALSE}
prp(m4)
```






## Scratch Pad
```{r}
aa = matrix(as.integer(rnorm(16,212,10)),byrow=T,nrow=4)
colnames(aa) = LETTERS[1:4]

bb = matrix(as.integer(runif(16,0,50)),byrow=T,nrow=4)
colnames(bb) = LETTERS[23:26]

cc = data.frame(foo=c("foo","bar","baz","beep"))
dd = cbind(cc,aa,bb)

```