##  R script to create term-document matrix
##  
##  Eryk Wdowiak
##  original:  23 Jun 2015
##  modified:  30 Dec 2018

##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##

##  load "tm" (text mining) package
library( tm )

##  ##  ##  ##  ##  ##  ##  ##  ##  ##

docs <- c("I spent the weekend with my niece.",
          "My niece likes swimming.",
          "My niece and I went swimming last weekend.",
          "We will go swimming this weekend and next weekend.")
doc_id <- paste("doc0", 1:4 , sep = "")
docs <- data.frame( doc_id = doc_id, text = docs ) 

##  make text lower case
docs$text <- tolower( docs$text )

##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##
  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##
##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##

##  FUNCTIONS
##  =========

##  make corpus
mkCorpus <- function( df ) {

  ##  create corpus from data frame
  corpus <- Corpus( DataframeSource( df ))

  ##  clean it up
  corpus <- tm_map( corpus, removePunctuation )
  corpus <- tm_map( corpus, removeNumbers )
  corpus <- tm_map( corpus, removeWords, stopwords("english") )
  corpus <- tm_map( corpus, stripWhitespace )
  #corpus <- tm_map( corpus, stemDocument )
  corpus
}

##  make term-document matrix
mkDocTerm <- function( df ) {
    
    ##  create it from data frame
    corpus <- mkCorpus( df )
    tdm <- DocumentTermMatrix( corpus )
    tdm
}

##  compute cosine measure
mkCosine <- function( tdm ) {

    ##  interpret the matrix as a matrix
    tdm <- as.matrix( tdm )

    ##  compute dot product
    dot <- tdm %*% t(tdm)
    
    ##  cosine measure = dot product / product of lengths
    csim <- dot / sqrt( diag( dot ) %*% t(diag( dot )) )
    csim
}


##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##
  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##
##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##  ##

##  CORRELATIONS
##  ============

##  create document term matrix 
docTerms <- mkDocTerm( df = docs )
as.matrix( docTerms )

##  compute cosine measure of similarity between documents
csim <- mkCosine( tdm = docTerms )
round( csim , 3 ) 

##  find terms used at least 3 times 
findFreqTerms(docTerms, lowfreq=3)

##  what did you and Lauren talk about when you used the word "niece?"
##  find terms that correlate with "niece" by more than 0.10
findAssocs(docTerms, term = "niece" , corlimit = 0.10)


##  inspect the document term matrix
as.matrix( docTerms[,c("likes","niece","swimming","weekend")] ) 

##  get the correlation coefficients
round( cor( as.matrix( docTerms[,c("likes","niece","swimming","weekend")] ) ),3)


##  exit without saving data
q("no")