17public Word() => _symbols = new Vec<Symbol>(); 25_symbols = new Vec<Symbol>(capacity);

30public int SymbolsCount => _symbols.Count; 37int len = _symbols.Count; 42_symbols[len - 1].Next = len; 46_symbols.Push(new Symbol(c, prev, next, charLength)); 56if (i >= _symbols.Count) 62if (_symbols[i].C == c1 && i + 1 < _symbols.Count && _symbols[i + 1].C == c2) 64Symbol first = _symbols[i]; 65Symbol second = _symbols[i + 1]; 70changes.Push((Pair<int>.Create(_symbols[i - 1].C, first.C), -1)); 71changes.Push((Pair<int>.Create(_symbols[i - 1].C, replacement), 1)); 79_symbols[i].C = replacement; 80_symbols[i].Prev = first.Prev; 81_symbols[i].Next = second.Next; 82_symbols[i].Len = first.Len + second.Len; 84_symbols.Remove(i + 1); 87if (i < _symbols.Count - 1) 89changes.Push((Pair<int>.Create(second.C, _symbols[i + 1].C), -1)); 90changes.Push((Pair<int>.Create(replacement, _symbols[i + 1].C), 1)); 102priorityQueue ??= new PriorityQueue<Merge>(_symbols.Count); 107for (int i = 0; i < _symbols.Count - 1; i++) 109if (merges.TryGetValue(Pair<int>.Create(_symbols[i].C, _symbols[i + 1].C), out (int m1, int m2) value)) 132if (_symbols.Count == 0 || _symbols[top.Pos].Len == 0 || _symbols[top.Pos].Next == -1) 137int nextPos = _symbols[top.Pos].Next; 138Symbol right = _symbols[nextPos]; 141Pair<int> targetNewPair = Pair<int>.Create(_symbols[top.Pos].C, right.C); 148_symbols[top.Pos].MergeWith(ref right, top.NewId); 151_symbols[nextPos].Len = 0; 154if (right.Next > -1 && right.Next < _symbols.Count) 156_symbols[right.Next].Prev = top.Pos; 160Symbol current = _symbols[top.Pos]; 164Symbol prevSymbol = _symbols[prev]; 175if ((uint)next < (uint)_symbols.Count) 177Symbol nextSymbol = _symbols[next]; 188for (int i = _symbols.Count - 1; i >= 0; i--) 190if (_symbols[i].Len == 0) 192_symbols.Remove(i); 201accumulatedIds.Add(_symbols[i].C); 213accumulatedIds.Add(_symbols[i].C); 214charsConsumed += _symbols[i].Len; 228accumulatedIds.Add(_symbols[i].C); 229textIndex -= _symbols[i].Len; 243charsConsumed += _symbols[i].Len; 257textIndex -= _symbols[i].Len; 266for (int i = 0; i < _symbols.Count; i++) 268chars.Push(_symbols[i].C); 276if (_symbols.Count == 0) 283sb.Append($"{_symbols[0].C}"); 284for (int i = 1; i < _symbols.Count; i++) 286sb.Append($", {_symbols[i].C}"); 298int endIndex = index + _symbols[i].Len; 299tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len))); 300index += _symbols[i].Len;