| File: Expression\KeyWordTable.cs | Web Access |
| Project: src\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj (Microsoft.ML.Transforms) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Collections.Generic; using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Runtime; namespace Microsoft.ML.Transforms { /// <summary> /// Maps from normalized string to keyword token kind. A lexer must be provided with one of these. /// </summary> internal partial class KeyWordTable { public struct KeyWordKind { public readonly TokKind Kind; public readonly bool IsContextKeyWord; public KeyWordKind(TokKind kind, bool isContextKeyWord) { Kind = kind; IsContextKeyWord = isContextKeyWord; } } private readonly NormStr.Pool _pool; private readonly Dictionary<NormStr, KeyWordKind> _mpnstrtidWord; private readonly Dictionary<NormStr, TokKind> _mpnstrtidPunc; public KeyWordTable(NormStr.Pool pool) { Contracts.AssertValue(pool); _pool = pool; _mpnstrtidWord = new Dictionary<NormStr, KeyWordKind>(); _mpnstrtidPunc = new Dictionary<NormStr, TokKind>(); } public void AddKeyWord(string str, TokKind tid) { Contracts.AssertNonEmpty(str); _mpnstrtidWord.Add(_pool.Add(str), new KeyWordKind(tid, false)); } public bool TryAddPunctuator(string str, TokKind tid) { Contracts.AssertNonEmpty(str); // Note: this assumes that once a prefix is found, that all shorter // prefixes are mapped to something (TokKind.None to indicate that // it is only a prefix and not itself a token). TokKind tidCur; NormStr nstr = _pool.Add(str); if (_mpnstrtidPunc.TryGetValue(_pool.Add(str), out tidCur)) { if (tidCur == tid) return true; if (tidCur != TokKind.None) return false; } else { // Map all prefixes (that aren't already mapped) to TokKind.None. for (int cch = str.Length; --cch > 0;) { NormStr nstrTmp = _pool.Add(str.Substring(0, cch)); TokKind tidTmp; if (_mpnstrtidPunc.TryGetValue(_pool.Add(nstrTmp.Value), out tidTmp)) break; _mpnstrtidPunc.Add(nstrTmp, TokKind.None); } } _mpnstrtidPunc[nstr] = tid; return true; } public void AddPunctuator(string str, TokKind tid) { Contracts.AssertNonEmpty(str); if (!TryAddPunctuator(str, tid)) Contracts.Assert(false, "duplicate punctuator!"); } public bool IsKeyWord(NormStr nstr, out KeyWordKind kind) { Contracts.Assert(!nstr.Value.IsEmpty); return _mpnstrtidWord.TryGetValue(nstr, out kind); } public bool IsPunctuator(NormStr nstr, out TokKind tid) { Contracts.Assert(!nstr.Value.IsEmpty); return _mpnstrtidPunc.TryGetValue(nstr, out tid); } public IEnumerable<KeyValuePair<NormStr, TokKind>> Punctuators { get { return _mpnstrtidPunc; } } public IEnumerable<KeyValuePair<NormStr, KeyWordKind>> KeyWords { get { return _mpnstrtidWord; } } } }