Skip to content

A possible way to deal with elisions #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
odelmarcelle wants to merge 3 commits into
base: master
Choose a base branch
from
Open

A possible way to deal with elisions #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
af6d016
remove_elisions argument
odelmarcelle Dec 3, 2021
56b777f
fix utf8 documentation
odelmarcelle Dec 3, 2021
8479514
fix utf8 regex
odelmarcelle Dec 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 30 additions & 20 deletions R/sentiment_engines.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,21 @@ spread_sentiment_features <- function(s, features, lexNames) {
s[]
}

tokenize_texts <- function(x, tokens = NULL, type = "word") { # x embeds a character vector
tokenize_texts <- function(x, tokens = NULL, type = "word", remove_elisions = TRUE) { # x embeds a character vector
if (is.null(tokens)) {
if (type == "word") {
lower <- stringi::stri_trans_tolower(x)
if (remove_elisions) lower <- stringi::stri_replace_all(lower, "", regex = "\\b([lmtnsjdc]|qu|jusqu|quoiqu|lorsqu|puisqu)[\\'\\u2019]")
tok <- stringi::stri_split_boundaries(
stringi::stri_trans_tolower(x),
type = "word", skip_word_none = TRUE, skip_word_number = TRUE
lower, type = "word", skip_word_none = TRUE, skip_word_number = TRUE
)
} else if (type == "sentence") {
sentences <- stringi::stri_split_boundaries(x, type = "sentence")
tok <- lapply(sentences, function(sn) { # list of documents of list of sentences of words
lower <- stringi::stri_trans_tolower(gsub(", ", " c_c ", sn)) # reset commas
if (remove_elisions) lower <- stringi::stri_replace_all(lower, "", regex = "\\b([lmtnsjdc]|qu|jusqu|quoiqu|lorsqu|puisqu)[\\'\\u2019]")
wo <- stringi::stri_split_boundaries(
stringi::stri_trans_tolower(gsub(", ", " c_c ", sn)), # reset commas
lower,
type = "word", skip_word_none = TRUE, skip_word_number = TRUE
)
wo[sapply(wo, length) != 0]
Expand All @@ -29,12 +32,12 @@ tokenize_texts <- function(x, tokens = NULL, type = "word") { # x embeds a chara
tok
}

compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence = FALSE, nCore = 1) {
compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
threads <- min(RcppParallel::defaultNumThreads(), nCore)
RcppParallel::setThreadOptions(numThreads = threads)
if (is_only_character(x)) x <- quanteda::corpus(x)
if (do.sentence == TRUE) {
tokens <- tokenize_texts(as.character(x), tokens, type = "sentence")
tokens <- tokenize_texts(as.character(x), tokens, type = "sentence", remove_elisions)
valenceType <- ifelse(is.null(lexicons[["valence"]]), 0,
ifelse(colnames(lexicons[["valence"]])[2] == "y", 1, 2))
s <- compute_sentiment_sentences(unlist(tokens, recursive = FALSE),
Expand All @@ -50,7 +53,7 @@ compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence
data.table::setcolorder(s, c("id", "sentence_id", "word_count"))
}
} else {
tokens <- tokenize_texts(as.character(x), tokens, type = "word")
tokens <- tokenize_texts(as.character(x), tokens, type = "word", remove_elisions)
if (is.null(lexicons[["valence"]])) { # call to C++ code
s <- compute_sentiment_onegrams(tokens, lexicons, how)
} else {
Expand All @@ -66,7 +69,7 @@ compute_sentiment_lexicons <- function(x, tokens, dv, lexicons, how, do.sentence
}

compute_sentiment_multiple_languages <- function(x, lexicons, languages, features, how,
tokens = NULL, do.sentence = FALSE, nCore = 1) {
tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
ids <- quanteda::docnames(x) # original ids to keep same order in output

# split corpus by language
Expand All @@ -82,7 +85,7 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature
if (!(l %in% names(lexicons))) {
stop(paste0("No lexicon found for language: ", l))
}
s <- compute_sentiment(corpus, lexicons[[l]], how, tokens[idxs[[l]]], do.sentence, nCore)
s <- compute_sentiment(corpus, lexicons[[l]], how, tokens[idxs[[l]]], do.sentence, nCore, remove_elisions)
sentByLang[[l]] <- s
}

Expand All @@ -94,6 +97,8 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature

#' Compute textual sentiment across features and lexicons
#'
#' @encoding UTF-8
#'
#' @author Samuel Borms, Jeroen Van Pelt, Andres Algaba
#'
#' @description Given a corpus of texts, computes sentiment per document or sentence using the valence shifting
Expand Down Expand Up @@ -146,6 +151,11 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature
#' computation only for a sufficiently large corpus.
#' @param do.sentence a \code{logical} to indicate whether the sentiment computation should be done on
#' sentence-level rather than document-level. By default \code{do.sentence = FALSE}.
#' @param remove_elisions a \code{logical} to indicate whether elisions should be removed in front of words. Elisions are
#' notably present in the French language where some words are for example preceded by "l'" or "d'", such as "l'amélioration".
#' Standard tokenization rules do not separate the elision from the word, which prevent the identification of some sentiment words.
#' If \code{TRUE}, removes the French elisions from all words prior to the sentiment computation. The removed elisions are:
#' "l'", "m'", "t'", "qu'", "n'", "s'", "j'", "d'", "c'", "jusqu'", "quoiqu'", "lorsqu'" and "puisqu'".
#'
#' @return If \code{x} is a \code{sento_corpus} object: a \code{sentiment} object, i.e., a \code{data.table} containing
#' the sentiment scores \code{data.table} with an \code{"id"}, a \code{"date"} and a \code{"word_count"} column,
Expand Down Expand Up @@ -226,7 +236,7 @@ compute_sentiment_multiple_languages <- function(x, lexicons, languages, feature
#'
#' @importFrom compiler cmpfun
#' @export
compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) {
compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
if (!(how %in% get_hows()[["words"]]))
stop("Please select an appropriate aggregation 'how'.")
if (length(nCore) != 1 || !is.numeric(nCore))
Expand Down Expand Up @@ -280,7 +290,7 @@ compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL,
UseMethod("compute_sentiment", x)
}

.compute_sentiment.sento_corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) {
.compute_sentiment.sento_corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
nCore <- check_nCore(nCore)

languages <- tryCatch(unique(quanteda::docvars(x, field = "language")), error = function(e) NULL)
Expand All @@ -292,7 +302,7 @@ compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL,
s <- spread_sentiment_features(s, features, lexNames) # there is always at least one feature
} else {
features <- names(quanteda::docvars(x))[-c(1:2)] # drop date and language column
s <- compute_sentiment_multiple_languages(x, lexicons, languages, features, how, tokens, do.sentence, nCore)
s <- compute_sentiment_multiple_languages(x, lexicons, languages, features, how, tokens, do.sentence, nCore, remove_elisions)
}

s <- s[order(date)] # order by date
Expand All @@ -304,7 +314,7 @@ compute_sentiment <- function(x, lexicons, how = "proportional", tokens = NULL,
#' @export
compute_sentiment.sento_corpus <- compiler::cmpfun(.compute_sentiment.sento_corpus)

.compute_sentiment.corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) {
.compute_sentiment.corpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
nCore <- check_nCore(nCore)

if (ncol(quanteda::docvars(x)) == 0) {
Expand All @@ -316,7 +326,7 @@ compute_sentiment.sento_corpus <- compiler::cmpfun(.compute_sentiment.sento_corp
dv <- data.table::as.data.table(quanteda::docvars(x)[features])
}

s <- compute_sentiment_lexicons(x, tokens, dv, lexicons, how, do.sentence, nCore)
s <- compute_sentiment_lexicons(x, tokens, dv, lexicons, how, do.sentence, nCore, remove_elisions)

if (!is.null(features)) { # spread sentiment across numeric features if present and reformat
lexNames <- names(lexicons)[names(lexicons) != "valence"]
Expand All @@ -330,9 +340,9 @@ compute_sentiment.sento_corpus <- compiler::cmpfun(.compute_sentiment.sento_corp
#' @export
compute_sentiment.corpus <- compiler::cmpfun(.compute_sentiment.corpus)

.compute_sentiment.character <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) {
.compute_sentiment.character <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
nCore <- check_nCore(nCore)
s <- compute_sentiment_lexicons(x, tokens, dv = NULL, lexicons, how, do.sentence, nCore)
s <- compute_sentiment_lexicons(x, tokens, dv = NULL, lexicons, how, do.sentence, nCore, remove_elisions)

s
}
Expand All @@ -341,18 +351,18 @@ compute_sentiment.corpus <- compiler::cmpfun(.compute_sentiment.corpus)
#' @export
compute_sentiment.character <- compiler::cmpfun(.compute_sentiment.character)

.compute_sentiment.VCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) {
.compute_sentiment.VCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
compute_sentiment(unlist(lapply(x, "[[", "content")),
lexicons, how, tokens, do.sentence, nCore)
lexicons, how, tokens, do.sentence, nCore, remove_elisions)
}

#' @importFrom compiler cmpfun
#' @export
compute_sentiment.VCorpus <- compiler::cmpfun(.compute_sentiment.VCorpus)

.compute_sentiment.SimpleCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1) {
.compute_sentiment.SimpleCorpus <- function(x, lexicons, how = "proportional", tokens = NULL, do.sentence = FALSE, nCore = 1, remove_elisions = TRUE) {
compute_sentiment(as.character(as.list(x)),
lexicons, how, tokens, do.sentence, nCore)
lexicons, how, tokens, do.sentence, nCore, remove_elisions)
}

#' @importFrom compiler cmpfun
Expand Down
10 changes: 9 additions & 1 deletion man/compute_sentiment.Rd
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions tests/testthat/test_sentiment_computation.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,3 +207,27 @@ test_that("Same tf-idf scoring for sentometrics and quanteda", {
unname(posScores - negScores))
})

# elision check
test_that("Sentiment words are recognized after elision removal", {
sample_text <- setNames(nm = c("Je me sens abandonné.", "J'ai peur de l'abandon",
"J'ai peur de l’abandon", "C'est un abandon",
"J'ai une sensation aiguë d'abandon"))
s1 <- compute_sentiment(
sample_text,
lexicons = sentometrics::sento_lexicons(list(LM_sample = head(sentometrics::list_lexicons$LM_fr_tr, 5)),
list_valence_shifters$fr),
how = "proportionalPol"
)
s2 <- compute_sentiment(
sample_text,
lexicons = sentometrics::sento_lexicons(list(LM_sample = head(sentometrics::list_lexicons$LM_fr_tr, 5)),
list_valence_shifters$fr),
how = "proportionalPol",
remove_elisions = FALSE
)

expect_equal(s1$LM_sample, c(-1, -1, -1, -1, -1.8))
expect_equal(s2$LM_sample, c(-1, 0, 0, -1, 0))

})