From 17c8b184c4116bf540c7d8e9aad596e36c5e809d Mon Sep 17 00:00:00 2001 From: diego Date: Wed, 9 Oct 2013 08:52:50 +0800 Subject: [PATCH 1/8] START work on a string EXISTS feature..no need for replacement, returns true/1 on first find --- boxwood.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++ php_boxwood.c | 60 +++++++++++++++++++++++++++++++----- php_boxwood.h | 1 + 3 files changed, 138 insertions(+), 8 deletions(-) diff --git a/boxwood.c b/boxwood.c index 821f47f..b76b9a3 100644 --- a/boxwood.c +++ b/boxwood.c @@ -392,6 +392,69 @@ static int bw_replace_proper(struct bw_node_t *root, byte *modified_bytes, byte return len - shrunk_bytes; +} + + /* + start at root of trie and first byte + if byte is not in trie, go to next byte and repeat + if byte is in trie, save position as bad_word_start_maybe, advance byte and advance down trie + while byte is in trie, continue advancing each + if we get to the point where there is nothing in the next[] array on the trie node, we've reached the end of the + word, so replace from bad_word_start_maybe+1 to current byte with *s, and resume with root of trie and next byte + if we get to the point where the next[] array is not empty but does not contain an entry for the current byte + then we've diverged from the stem that matches, so don't do any replacement and start re-parsing at + bad_word_start_maybe+1 + + */ +static int bw_exists_proper(struct bw_node_t *root, byte *bytes_to_walk, int len, int check_multibyte, byte *word_boundary_chars) { + int match_start_maybe, i = 0, shrunk_bytes = 0, previous_terminal = 0; + int endok = 0; + struct bw_node_t *current_node; + + while (i < len) { + if ((! bw_node_has_next(root, bytes_to_walk[i])) // no matches at start + || (! boundary_before_word(word_boundary_chars, i, bytes_to_walk))) + { + i++; + } + else { + match_start_maybe = i; + current_node = root; + previous_terminal = 0; + do { + current_node = current_node->next[bytes_to_walk[i]]; + i++; + if (current_node->is_terminal + && boundary_after_word(word_boundary_chars, i, len, bytes_to_walk)) + { + previous_terminal = i; + } + } + while (bw_node_has_next(current_node, bytes_to_walk[i])); + + /* If we've stopped advancing because we've reached a character in the + * text which doesn't match the next byte in the trie -- then check to + * see if there was a previous node on our traversal that was marked + * as terminal -- that means even though we didn't match the longer + * word, there was a previous shorter prefix that we should treat as + * a match + */ + if ((! current_node->is_terminal) && previous_terminal) { + i = previous_terminal; + } + + /* if we've reached the end of the word, replace */ + if (((! current_node->has_next) && boundary_after_word(word_boundary_chars, i, len, bytes_to_walk)) || previous_terminal) { + return 1; + } + else { /* diverging from stem, start reparsing */ + i = match_start_maybe + 1; + } + } + } + + return 0; + } /** @@ -425,6 +488,28 @@ byte *bw_replace_text(struct bw_trie_t *trie, byte *text, byte replacement, int return modified_text; } +/* +* Return 1 is any are found, 0 for false +*/ +int bw_exists_text(struct bw_trie_t *trie, byte *text, int wordbound) { + int text_len = strlen((char *)text); + int result = 0; + + byte *text_to_walk; + if (trie->case_insensitive && trie->folding_trie) { + text_to_walk = case_fold_lower(trie->folding_trie, text, text_len); + } else { + text_to_walk = text; + } + + result = bw_exists_proper(trie->root, text_to_walk, text_len, 1, wordbound ? trie->word_boundary_chars : NULL); + + if (text_to_walk != text) { + free(text_to_walk); + } + + return result; +} /** * Replace sub-sequences in the byte sequence that match words added to the trie. No diff --git a/php_boxwood.c b/php_boxwood.c index 5b31c47..a8f61d4 100644 --- a/php_boxwood.c +++ b/php_boxwood.c @@ -35,6 +35,7 @@ zend_function_entry boxwood_functions[] = { PHP_FE(boxwood_add_text, NULL) PHP_FE(boxwood_replace_text, NULL) PHP_FE(boxwood_set_word_boundary_bytes, NULL) + PHP_FE(boxwood_exists, NULL) {NULL, NULL, NULL} /* Must be the last line in boxwood_functions[] */ }; /* }}} */ @@ -235,11 +236,54 @@ PHP_FUNCTION(boxwood_set_word_boundary_bytes) bw_set_word_boundary_bytes(trie, text); } -/* - * Local variables: - * tab-width: 4 - * c-basic-offset: 4 - * End: - * vim600: noet sw=4 ts=4 fdm=marker - * vim<600: noet sw=4 ts=4 - */ +PHP_FUNCTION(boxwood_exists) +{ + struct bw_trie_t *trie; + zval *znode; + zval *ztext; + zval *zwordbound; + int result = 0; + + int wordbound = 0; + + int num_args = ZEND_NUM_ARGS(); + + if (num_args == 2) { + if (zend_parse_parameters(num_args TSRMLS_CC, "zz", &znode, &ztext, &zreplacement) == FAILURE) { + WRONG_PARAM_COUNT; + } + } else if (num_args == 3) { + if (zend_parse_parameters(num_args TSRMLS_CC, "zzz", &znode, &ztext,&zwordbound) == FAILURE) { + WRONG_PARAM_COUNT; + } + } else { + WRONG_PARAM_COUNT; + } + + if (Z_TYPE_P(znode) != IS_RESOURCE) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "Parameter mismatch: first argument must be a Boxwood resource"); + RETURN_FALSE; + } + if (Z_TYPE_P(ztext) != IS_STRING) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "Parameter mismatch: second argument must be a string"); + RETURN_FALSE; + } + + if (num_args == 3) { + if (Z_TYPE_P(zwordbound) != IS_BOOL) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "Parameter mismatch: fourth argument must be a boolean"); + RETURN_FALSE; + } + wordbound = Z_BVAL_P(zwordbound) ? 1 : 0; + } + + ZEND_FETCH_RESOURCE(trie, struct bw_trie_t*, &znode, -1, PHP_BOXWOOD_TRIE_RES_NAME, le_bw_trie); + + if (Z_TYPE_P(ztext) == IS_STRING) { + result = bw_exists_text(trie, (byte *) Z_STRVAL_P(ztext), wordbound); + if(result == 1) RETURN_TRUE; + else RETURN_FALSE; + } + + RETURN_FALSE: +} diff --git a/php_boxwood.h b/php_boxwood.h index 1308e88..04a5e03 100644 --- a/php_boxwood.h +++ b/php_boxwood.h @@ -39,6 +39,7 @@ PHP_FUNCTION(boxwood_new); PHP_FUNCTION(boxwood_add_text); PHP_FUNCTION(boxwood_replace_text); PHP_FUNCTION(boxwood_set_word_boundary_bytes); +PHP_FUNCTION(boxwood_exists); ZEND_BEGIN_MODULE_GLOBALS(boxwood) struct case_fold_branch_t *folding_trie; From e4cd576400f18ad84ee1f829c676d4ad75d1e209 Mon Sep 17 00:00:00 2001 From: diego Date: Wed, 9 Oct 2013 08:57:42 +0800 Subject: [PATCH 2/8] REMOVE replacement var in exists fun --- php_boxwood.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php_boxwood.c b/php_boxwood.c index a8f61d4..def07de 100644 --- a/php_boxwood.c +++ b/php_boxwood.c @@ -249,7 +249,7 @@ PHP_FUNCTION(boxwood_exists) int num_args = ZEND_NUM_ARGS(); if (num_args == 2) { - if (zend_parse_parameters(num_args TSRMLS_CC, "zz", &znode, &ztext, &zreplacement) == FAILURE) { + if (zend_parse_parameters(num_args TSRMLS_CC, "zz", &znode, &ztext) == FAILURE) { WRONG_PARAM_COUNT; } } else if (num_args == 3) { From 503383b5b5e956483d6c5545b3eb14cd1b77179d Mon Sep 17 00:00:00 2001 From: diego Date: Wed, 9 Oct 2013 08:59:18 +0800 Subject: [PATCH 3/8] typo --- php_boxwood.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/php_boxwood.c b/php_boxwood.c index def07de..6e4ccaf 100644 --- a/php_boxwood.c +++ b/php_boxwood.c @@ -281,8 +281,12 @@ PHP_FUNCTION(boxwood_exists) if (Z_TYPE_P(ztext) == IS_STRING) { result = bw_exists_text(trie, (byte *) Z_STRVAL_P(ztext), wordbound); - if(result == 1) RETURN_TRUE; - else RETURN_FALSE; + if(result == 1) { + RETURN_TRUE; + } + else { + RETURN_FALSE; + } } RETURN_FALSE: From 9d28e0db8b9b87429cb17acda96b8c4fb622ed70 Mon Sep 17 00:00:00 2001 From: diego Date: Wed, 9 Oct 2013 09:00:11 +0800 Subject: [PATCH 4/8] more typo --- php_boxwood.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php_boxwood.c b/php_boxwood.c index 6e4ccaf..ad0d4bc 100644 --- a/php_boxwood.c +++ b/php_boxwood.c @@ -289,5 +289,5 @@ PHP_FUNCTION(boxwood_exists) } } - RETURN_FALSE: + RETURN_FALSE; } From c900a0f4c2b7b2d1943a67e6df32754c494b5c2b Mon Sep 17 00:00:00 2001 From: diego Date: Wed, 9 Oct 2013 09:25:08 +0800 Subject: [PATCH 5/8] ADD tests for exists --- tests/014.phpt | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/014.phpt diff --git a/tests/014.phpt b/tests/014.phpt new file mode 100644 index 0000000..0ddf472 --- /dev/null +++ b/tests/014.phpt @@ -0,0 +1,30 @@ +--TEST-- +Test EXISTS function +--SKIPIF-- + +--FILE-- + +--EXPECT-- +yes +yes +no +no +yes +yes From 18bf9ca43b6032936343c75aea3202c304dd502b Mon Sep 17 00:00:00 2001 From: diego Date: Wed, 9 Oct 2013 09:26:50 +0800 Subject: [PATCH 6/8] typo --- tests/014.phpt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/014.phpt b/tests/014.phpt index 0ddf472..7af4511 100644 --- a/tests/014.phpt +++ b/tests/014.phpt @@ -18,7 +18,7 @@ $tests = array( ); foreach ($tests as $test) { - echo (boxwood_exists($r, $test) === true ? "yes" : no), "\n"; + echo (boxwood_exists($r, $test) === true ? "yes" : "no"), "\n"; } ?> --EXPECT-- From b01044f945055e5a33d5b11b7ba9969899fbc2c9 Mon Sep 17 00:00:00 2001 From: diego Date: Wed, 9 Oct 2013 09:28:44 +0800 Subject: [PATCH 7/8] FIX test 014 --- tests/014.phpt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/014.phpt b/tests/014.phpt index 7af4511..decd5b3 100644 --- a/tests/014.phpt +++ b/tests/014.phpt @@ -9,12 +9,12 @@ boxwood_add_text($r, "airplane"); boxwood_add_text($r, "train"); $tests = array( - 'my train airplane norway', - 'my norway', + 'airplane norway', + 'my norway train', 'toast ting', 'no way', - 'drive my norway', - 'fly airplane' + 'drive my train to norway', + 'fly airplane to tokyo' ); foreach ($tests as $test) { From bd4fa5996f1e21c50e9b9f54e6dc6e343b8d40ec Mon Sep 17 00:00:00 2001 From: diego Date: Thu, 26 Dec 2013 19:27:52 +0800 Subject: [PATCH 8/8] 1) Add the ability to pass array of string to add_text() 2) allow array of text to be passed to exists() and return on first positive 3) add php tests --- php_boxwood.c | 47 +++++++++++++++++++++++++++++++++++++++-------- tests/014.phpt | 7 +++++-- tests/015.phpt | 29 +++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 tests/015.phpt diff --git a/php_boxwood.c b/php_boxwood.c index ad0d4bc..4a53e3b 100644 --- a/php_boxwood.c +++ b/php_boxwood.c @@ -121,18 +121,33 @@ PHP_FUNCTION(boxwood_add_text) { struct bw_trie_t *trie; zval *znode; - char *text; - int text_len; + zval *ztext; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "rs", &znode, &text, &text_len) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "rz", &znode, &ztext) == FAILURE) { RETURN_FALSE; } ZEND_FETCH_RESOURCE(trie, struct bw_trie_t*, &znode, -1, PHP_BOXWOOD_TRIE_RES_NAME, le_bw_trie); - int added = bw_add_text(trie, (byte *) text); - RETURN_LONG(added); + int added = 0; + + if (Z_TYPE_P(ztext) == IS_STRING) { + added = bw_add_text(trie, (byte *) Z_STRVAL_P(ztext)); + } + else if (Z_TYPE_P(ztext) == IS_ARRAY) { + zval **one_text; + HashTable *arr = Z_ARRVAL_P(ztext); + HashPosition ptr; + + for (zend_hash_internal_pointer_reset_ex(arr, &ptr); + zend_hash_get_current_data_ex(arr, (void **) &one_text, &ptr) == SUCCESS; + zend_hash_move_forward_ex(arr, &ptr)) { + + added += bw_add_text(trie, (byte *) Z_STRVAL_PP(one_text)); + } + } + RETURN_LONG(added); } PHP_FUNCTION(boxwood_replace_text) @@ -242,7 +257,7 @@ PHP_FUNCTION(boxwood_exists) zval *znode; zval *ztext; zval *zwordbound; - int result = 0; + int result; int wordbound = 0; @@ -264,8 +279,8 @@ PHP_FUNCTION(boxwood_exists) php_error_docref(NULL TSRMLS_CC, E_ERROR, "Parameter mismatch: first argument must be a Boxwood resource"); RETURN_FALSE; } - if (Z_TYPE_P(ztext) != IS_STRING) { - php_error_docref(NULL TSRMLS_CC, E_ERROR, "Parameter mismatch: second argument must be a string"); + if (Z_TYPE_P(ztext) != IS_STRING && Z_TYPE_P(ztext) != IS_ARRAY) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "Parameter mismatch: second argument must be a string or array"); RETURN_FALSE; } @@ -288,6 +303,22 @@ PHP_FUNCTION(boxwood_exists) RETURN_FALSE; } } + else if (Z_TYPE_P(ztext) == IS_ARRAY) { + zval **one_text; + HashTable *arr = Z_ARRVAL_P(ztext); + HashPosition ptr; + for (zend_hash_internal_pointer_reset_ex(arr, &ptr); + zend_hash_get_current_data_ex(arr, (void **) &one_text, &ptr) == SUCCESS; + zend_hash_move_forward_ex(arr, &ptr)) { + + result = bw_exists_text(trie, (byte *) Z_STRVAL_PP(one_text), wordbound); + if(result == 1) { + RETURN_TRUE; + } + } + + RETURN_FALSE; + } RETURN_FALSE; } diff --git a/tests/014.phpt b/tests/014.phpt index decd5b3..a4460ab 100644 --- a/tests/014.phpt +++ b/tests/014.phpt @@ -8,18 +8,20 @@ $r = boxwood_new(); boxwood_add_text($r, "airplane"); boxwood_add_text($r, "train"); -$tests = array( +$tests = [ 'airplane norway', 'my norway train', 'toast ting', 'no way', 'drive my train to norway', 'fly airplane to tokyo' -); +]; foreach ($tests as $test) { echo (boxwood_exists($r, $test) === true ? "yes" : "no"), "\n"; } + +echo (boxwood_exists($r, $tests) === true ? "yes" : "no"), "\n"; ?> --EXPECT-- yes @@ -28,3 +30,4 @@ no no yes yes +yes \ No newline at end of file diff --git a/tests/015.phpt b/tests/015.phpt new file mode 100644 index 0000000..d4a34b9 --- /dev/null +++ b/tests/015.phpt @@ -0,0 +1,29 @@ +--TEST-- +Test Multi Add Text function +--SKIPIF-- + +--FILE-- + +--EXPECT-- +yes +yes +no +no +yes +yes