diff --git a/R/sentiment_engines.R b/R/sentiment_engines.R index 17dfbc9..1be82b3 100644 --- a/R/sentiment_engines.R +++ b/R/sentiment_engines.R @@ -8,18 +8,21 @@ spread_sentiment_features <- function(s, features, lexNames) { s[] } -tokenize_texts <- function(x, tokens = NULL, type = "word") { # x embeds a character vector +tokenize_texts <- function(x, tokens = NULL, type = "word", remove_elisions = TRUE) { # x embeds a character vector if (is.null(tokens)) { if (type == "word") { + lower <- stringi::stri_trans_tolower(x) + if (remove_elisions) lower <- stringi::stri_replace_all(lower, "", regex = "\\b([lmtnsjdc]|qu|jusqu|quoiqu|lorsqu|puisqu)[\\'\\u2019]") tok <- stringi::stri_split_boundaries( - stringi::stri_trans_tolower(x), - type = "word", skip_word_none = TRUE, skip_word_number = TRUE + lower, type = "word", skip_word_none = TRUE, skip_word_number = TRUE ) } else if (type == "sentence") { sentences <- stringi::stri_split_boundaries(x, type = "sentence") tok <- lapply(sentences, function(sn) { # list of documents of list of sentences of words + lower <- stringi::stri_trans_tolower(gsub(", ", " c_c ", sn)) # reset commas + if (remove_elisions) lower <- stringi::stri_replace_all(lower, "", regex = "\\b([lmtnsjdc]|qu|jusqu|quoiqu|lorsqu|puisqu)[\\'\\u2019]") wo <- stringi::stri_split_boundaries( - stringi::stri_trans_tolower(gsub(", ", " c_c ", sn)), # reset commas + lower, type = "word", skip_word_none = TRUE, skip_word_number = TRUE ) wo[sapply(wo, length) != 0] @@ -29,12 +32,12 @@ tokenize_texts <- function(x, tokens = NULL, type = "word") { # x embeds a chara tok } -compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence = FALSE, nCore = 1) { +compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { threads <- min(RcppParallel::defaultNumThreads(), nCore) RcppParallel::setThreadOptions(numThreads = threads) if (is_only_character(x)) x <- quanteda::corpus(x) if (do.sentence == TRUE) { - tokens <- tokenize_texts(as.character(x), tokens, type = "sentence") + tokens <- tokenize_texts(as.character(x), tokens, type = "sentence", remove_elisions) valenceType <- ifelse(is.null(lexicons[["valence"]]), 0, ifelse(colnames(lexicons[["valence"]])[2] == "y", 1, 2)) s <- compute_sentiment_sentences(unlist(tokens, recursive = FALSE), @@ -50,7 +53,7 @@ compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence data.table::setcolorder(s, c("id", "sentence_id", "word_count")) } } else { - tokens <- tokenize_texts(as.character(x), tokens, type = "word") + tokens <- tokenize_texts(as.character(x), tokens, type = "word", remove_elisions) if (is.null(lexicons[["valence"]])) { # call to C++ code s <- compute_sentiment_onegrams(tokens, lexicons, how) } else { @@ -66,7 +69,7 @@ compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence } compute_sentiment_multiple_languages <- function(x, lexicons, languages, features, how, - tokens = NULL, do.sentence = FALSE, nCore = 1) { + tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { ids <- quanteda::docnames(x) # original ids to keep same order in output # split corpus by language @@ -82,7 +85,7 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature if (!(l %in% names(lexicons))) { stop(paste0("No lexicon found for language: ", l)) } - s <- compute_sentiment(corpus, lexicons[[l]], how, tokens[idxs[[l]]], do.sentence, nCore) + s <- compute_sentiment(corpus, lexicons[[l]], how, tokens[idxs[[l]]], do.sentence, nCore, remove_elisions) sentByLang[[l]] <- s } @@ -94,6 +97,8 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature #' Compute textual sentiment across features and lexicons #' +#' @encoding UTF-8 +#' #' @author Samuel Borms, Jeroen Van Pelt, Andres Algaba #' #' @description Given a corpus of texts, computes sentiment per document or sentence using the valence shifting @@ -146,6 +151,11 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature #' computation only for a sufficiently large corpus. #' @param do.sentence a \code{logical} to indicate whether the sentiment computation should be done on #' sentence-level rather than document-level. By default \code{do.sentence = FALSE}. +#' @param remove_elisions a \code{logical} to indicate whether elisions should be removed in front of words. Elisions are +#' notably present in the French language where some words are for example preceded by "l'" or "d'", such as "l'amélioration". +#' Standard tokenization rules do not separate the elision from the word, which prevent the identification of some sentiment words. +#' If \code{TRUE}, removes the French elisions from all words prior to the sentiment computation. The removed elisions are: +#' "l'", "m'", "t'", "qu'", "n'", "s'", "j'", "d'", "c'", "jusqu'", "quoiqu'", "lorsqu'" and "puisqu'". #' #' @return If \code{x} is a \code{sento_corpus} object: a \code{sentiment} object, i.e., a \code{data.table} containing #' the sentiment scores \code{data.table} with an \code{"id"}, a \code{"date"} and a \code{"word_count"} column, @@ -226,7 +236,7 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature #' #' @importFrom compiler cmpfun #' @export -compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) { +compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { if (!(how %in% get_hows()[["words"]])) stop("Please select an appropriate aggregation 'how'.") if (length(nCore) != 1 || !is.numeric(nCore)) @@ -280,7 +290,7 @@ compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL, UseMethod("compute_sentiment", x) } -.compute_sentiment.sento_corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) { +.compute_sentiment.sento_corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { nCore <- check_nCore(nCore) languages <- tryCatch(unique(quanteda::docvars(x, field = "language")), error = function(e) NULL) @@ -292,7 +302,7 @@ compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL, s <- spread_sentiment_features(s, features, lexNames) # there is always at least one feature } else { features <- names(quanteda::docvars(x))[-c(1:2)] # drop date and language column - s <- compute_sentiment_multiple_languages(x, lexicons, languages, features, how, tokens, do.sentence, nCore) + s <- compute_sentiment_multiple_languages(x, lexicons, languages, features, how, tokens, do.sentence, nCore, remove_elisions) } s <- s[order(date)] # order by date @@ -304,7 +314,7 @@ compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL, #' @export compute_sentiment.sento_corpus <- compiler::cmpfun(.compute_sentiment.sento_corpus) -.compute_sentiment.corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) { +.compute_sentiment.corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { nCore <- check_nCore(nCore) if (ncol(quanteda::docvars(x)) == 0) { @@ -316,7 +326,7 @@ compute_sentiment.sento_corpus <- compiler::cmpfun(.compute_sentiment.sento_corp dv <- data.table::as.data.table(quanteda::docvars(x)[features]) } - s <- compute_sentiment_lexicons(x, tokens, dv, lexicons, how, do.sentence, nCore) + s <- compute_sentiment_lexicons(x, tokens, dv, lexicons, how, do.sentence, nCore, remove_elisions) if (!is.null(features)) { # spread sentiment across numeric features if present and reformat lexNames <- names(lexicons)[names(lexicons) != "valence"] @@ -330,9 +340,9 @@ compute_sentiment.sento_corpus <- compiler::cmpfun(.compute_sentiment.sento_corp #' @export compute_sentiment.corpus <- compiler::cmpfun(.compute_sentiment.corpus) -.compute_sentiment.character <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) { +.compute_sentiment.character <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { nCore <- check_nCore(nCore) - s <- compute_sentiment_lexicons(x, tokens, dv = NULL, lexicons, how, do.sentence, nCore) + s <- compute_sentiment_lexicons(x, tokens, dv = NULL, lexicons, how, do.sentence, nCore, remove_elisions) s } @@ -341,18 +351,18 @@ compute_sentiment.corpus <- compiler::cmpfun(.compute_sentiment.corpus) #' @export compute_sentiment.character <- compiler::cmpfun(.compute_sentiment.character) -.compute_sentiment.VCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) { +.compute_sentiment.VCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { compute_sentiment(unlist(lapply(x, "[[", "content")), - lexicons, how, tokens, do.sentence, nCore) + lexicons, how, tokens, do.sentence, nCore, remove_elisions) } #' @importFrom compiler cmpfun #' @export compute_sentiment.VCorpus <- compiler::cmpfun(.compute_sentiment.VCorpus) -.compute_sentiment.SimpleCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) { +.compute_sentiment.SimpleCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) { compute_sentiment(as.character(as.list(x)), - lexicons, how, tokens, do.sentence, nCore) + lexicons, how, tokens, do.sentence, nCore, remove_elisions) } #' @importFrom compiler cmpfun diff --git a/man/compute_sentiment.Rd b/man/compute_sentiment.Rd index a2f4452..c0eaf01 100644 --- a/man/compute_sentiment.Rd +++ b/man/compute_sentiment.Rd @@ -1,5 +1,6 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/sentiment_engines.R +\encoding{UTF-8} \name{compute_sentiment} \alias{compute_sentiment} \title{Compute textual sentiment across features and lexicons} @@ -10,7 +11,8 @@ compute_sentiment( how = "proportional", tokens = NULL, do.sentence = FALSE, - nCore = 1 + nCore = 1, + remove_elisions = TRUE ) } \arguments{ @@ -39,6 +41,12 @@ sentence-level rather than document-level. By default \code{do.sentence = FALSE} \code{\link[RcppParallel]{setThreadOptions}} function, to parallelize the sentiment computation across texts. A value of 1 (default) implies no parallelization. Parallelization will improve speed of the sentiment computation only for a sufficiently large corpus.} + +\item{remove_elisions}{a \code{logical} to indicate whether elisions should be removed in front of words. Elisions are +notably present in the French language where some words are for example preceded by "l'" or "d'", such as "l'amélioration". +Standard tokenization rules do not separate the elision from the word, which prevent the identification of some sentiment words. +If \code{TRUE}, removes the French elisions from all words prior to the sentiment computation. The removed elisions are: +"l'", "m'", "t'", "qu'", "n'", "s'", "j'", "d'", "c'", "jusqu'", "quoiqu'", "lorsqu'" and "puisqu'".} } \value{ If \code{x} is a \code{sento_corpus} object: a \code{sentiment} object, i.e., a \code{data.table} containing diff --git a/tests/testthat/test_sentiment_computation.R b/tests/testthat/test_sentiment_computation.R index a9635a5..25f7eb0 100644 --- a/tests/testthat/test_sentiment_computation.R +++ b/tests/testthat/test_sentiment_computation.R @@ -207,3 +207,27 @@ test_that("Same tf-idf scoring for sentometrics and quanteda", { unname(posScores - negScores)) }) +# elision check +test_that("Sentiment words are recognized after elision removal", { + sample_text <- setNames(nm = c("Je me sens abandonné.", "J'ai peur de l'abandon", + "J'ai peur de l’abandon", "C'est un abandon", + "J'ai une sensation aiguë d'abandon")) + s1 <- compute_sentiment( + sample_text, + lexicons = sentometrics::sento_lexicons(list(LM_sample = head(sentometrics::list_lexicons$LM_fr_tr, 5)), + list_valence_shifters$fr), + how = "proportionalPol" + ) + s2 <- compute_sentiment( + sample_text, + lexicons = sentometrics::sento_lexicons(list(LM_sample = head(sentometrics::list_lexicons$LM_fr_tr, 5)), + list_valence_shifters$fr), + how = "proportionalPol", + remove_elisions = FALSE + ) + + expect_equal(s1$LM_sample, c(-1, -1, -1, -1, -1.8)) + expect_equal(s2$LM_sample, c(-1, 0, 0, -1, 0)) + +}) +