1 <?php
2 namespace apemsel\AttributedString;
3
4 /**
5 * Extends AttributedString to support a tokenized string.
6 *
7 * You can mix working with tokens with working on string ranges using the AttributedString methods.
8 * The original string is preserved.
9 *
10 * @author Adrian Pemsel <apemsel@gmail.com>
11 */
12 class TokenizedAttributedString extends AttributedString
13 {
14 protected $tokens;
15 protected $tokenOffsets;
16
17 /**
18 * @param string|AttributedString $string String to work on
19 * @param string $tokenizer Tokenizer to use, either "whitespace", "word" or a custom regex
20 */
21 public function __construct($string, $tokenizer = "whitespace", $attributeClass = "apemsel\AttributedString\BooleanArray") {
22 parent::__construct($string, $attributeClass);
23
24 $tokenizerFunction = "tokenizeOn".ucfirst($tokenizer);
25
26 if ($tokenizer[0] == "/") {
27 list($this->tokens, $this->tokenOffsets) = self::tokenizeOnRegex($string, $tokenizer);
28 } else {
29 if (!method_exists("apemsel\AttributedString\TokenizedAttributedString", $tokenizerFunction)) {
30 throw new \InvalidArgumentException("Unknown tokenizer $tokenizer");
31 }
32 list($this->tokens, $this->tokenOffsets) = self::$tokenizerFunction($string);
33 }
34
35 // convert byte to char offsets
36 $this->enableByteToCharCache();
37 $this->tokenOffsets = array_map(function($o) {
38 return $this->byteToCharOffset($o);
39 }, $this->tokenOffsets);
40 }
41
42 /**
43 * Return all tokens
44 *
45 * @return string[] tokens
46 */
47 public function getTokens() {
48 return $this->tokens;
49 }
50
51 /**
52 * Return all tokens' offsets
53 *
54 * @return int[] offsets
55 */
56 public function getTokenOffsets() {
57 return $this->tokenOffsets;
58 }
59
60 /**
61 * Return the number of tokens
62 *
63 * @return int count
64 */
65 public function getTokenCount() {
66 return count($this->tokens);
67 }
68
69 /**
70 * Get indicated token
71 *
72 * @param int $i token index
73 * @return string token
74 */
75 public function getToken($i) {
76 return $this->tokens[$i];
77 }
78
79 /**
80 * Get indicated token offset
81 *
82 * @param int $i token index
83 * @return int offset
84 */
85 public function getTokenOffset($i) {
86 return $this->tokenOffsets[$i];
87 }
88
89 /**
90 * Set a token to a given attribute and state
91 *
92 * @param int $i token index
93 * @param string $attribute attribute name
94 * @param bool $state attribute state
95 */
96 public function setTokenAttribute($i, $attribute, $state = true) {
97 $token = $this->tokens[$i];
98 $offset = $this->tokenOffsets[$i];
99 $length = mb_strlen($token, "utf-8");
100
101 return $this->setLength($offset, $length, $attribute, $state);
102 }
103
104 /**
105 * Set a range of tokens to a given attribute and state
106 *
107 * @param int $from token start index
108 * @param int $to token end index
109 * @param string $attribute attribute name
110 * @param bool $state attribute state
111 */
112 public function setTokenRangeAttribute($from, $to, $attribute, $state = true) {
113 $fromOffset = $this->tokenOffsets[$from];
114 $toOffset = $this->tokenOffsets[$to] + mb_strlen($this->tokens[$to], "utf-8") - 1;
115
116 return $this->setRange($fromOffset, $toOffset, $attribute, $state);
117 }
118
119 /**
120 * Set all tokens matching given dictionary to attribute and state
121 *
122 * @param string[] $dictionary dictionary
123 * @param string $attribute attribute name
124 * @param bool $state attribute state
125 */
126 public function setTokenDictionaryAttribute($dictionary, $attribute, $state = true) {
127 foreach($this->tokens as $i => $token) {
128 if (in_array($token, $dictionary)) {
129 $this->setTokenAttribute($i, $attribute, $state);
130 }
131 }
132 }
133
134 /**
135 * Get all attribute of token at given index
136 *
137 * @param int token index
138 * @return string[] attributes
139 */
140 public function attributesAtToken($i) {
141 return $this->attributesAt($this->tokenOffsets[$i]);
142 }
143
144 /**
145 * Convert all tokens to lower case
146 */
147 public function lowercaseTokens() {
148 $this->tokens = array_map(function($token) {
149 return mb_strtolower($token, "utf-8");
150 }, $this->tokens);
151 }
152
153 /**
154 * Tokenize a string on whitespace
155 *
156 * @param string $string string to be tokenized
157 * @return array array of two arrays, with tokens at index 0 and their byte offsets at index 1
158 */
159 public static function tokenizeOnWhitespace($string) {
160 // Matches pontential whitespace in front of the token and the token itself.
161 // Matching the whitespace could be omitted, but that results in slower execution ;-)
162 return self::tokenizeOnRegex($string, '/[\s\n\r]*([^\s\n\r]+)/u');
163 }
164
165 /**
166 * Tokenize a string on words
167 *
168 * @param string $string string to be tokenized
169 * @return array array of two arrays, with tokens at index 0 and their byte offsets at index 1
170 */
171 public static function tokenizeOnWords($string) {
172 return self::tokenizeOnRegex($string, '/([\p{L}\p{S}\p{N}]+)/u');
173 }
174
175 /**
176 * Tokenize a string with a given regex
177 *
178 * @param string $string string to be tokenized
179 * @param string $pattern regex. The token must be captured in the first subgroup.
180 * @return array array of two arrays, with tokens at index 0 and their byte offsets at index 1
181 */
182 public static function tokenizeOnRegex($string, $pattern)
183 {
184 // Fastest way to get both tokens and their offsets, but not easy to understand.
185 preg_match_all($pattern, $string, $matches, PREG_OFFSET_CAPTURE);
186
187 // $matches[1] contains an array of all matched subexpressions (= tokens)
188 // with their offset in column 1 and the matched token in column 0
189 $tokens = array_column($matches[1], 0);
190 $tokenOffsets = array_column($matches[1], 1);
191
192 return [$tokens, $tokenOffsets];
193 }
194
195 // Modified ArrayAccess interface
196
197 /**
198 * Check if the token at the given index exists
199 *
200 * @param int $i token index
201 * @return bool does the offset exist
202 */
203 public function offsetExists($i) {
204 return $i < $this->getTokenCount();
205 }
206
207 /**
208 * Get token at given index
209 *
210 * Note: TokenizedAttributedString uses the ArrayAccess interface to access tokens, not chars!
211 *
212 * @param int $i token index
213 * @return string token
214 */
215 public function offsetGet($i) {
216 return $this->tokens[$i];
217 }
218 }
219