TokenType

169 references to TokenType

System.Private.CoreLib (169)

src\runtime\src\libraries\System.Private.CoreLib\src\System\Globalization\DateTimeFormatInfo.cs (85)

1928if (IgnorableComma != sep) InsertHash(temp, IgnorableComma, TokenType.IgnorableSymbol, 0); 1929if (IgnorablePeriod != sep) InsertHash(temp, IgnorablePeriod, TokenType.IgnorableSymbol, 0); 1939InsertHash(temp, TimeSeparator, TokenType.SEP_Time, 0); 1944InsertHash(temp, " h", TokenType.SEP_HourSuff, 0); 1945InsertHash(temp, " h ", TokenType.SEP_HourSuff, 0); 1946InsertHash(temp, " min", TokenType.SEP_MinuteSuff, 0); 1947InsertHash(temp, " min ", TokenType.SEP_MinuteSuff, 0); 1948InsertHash(temp, " s", TokenType.SEP_SecondSuff, 0); 1949InsertHash(temp, " s ", TokenType.SEP_SecondSuff, 0); 1952InsertHash(temp, AMDesignator, TokenType.SEP_Am | TokenType.Am, 0); 1953InsertHash(temp, PMDesignator, TokenType.SEP_Pm | TokenType.Pm, 1); 1958InsertHash(temp, IgnorablePeriod + AMDesignator, TokenType.SEP_Am | TokenType.Am, 0); 1959InsertHash(temp, IgnorablePeriod + PMDesignator, TokenType.SEP_Pm | TokenType.Pm, 1); 1963InsertHash(temp, CJKYearSuff, TokenType.SEP_YearSuff, 0); 1964InsertHash(temp, KoreanYearSuff, TokenType.SEP_YearSuff, 0); 1965InsertHash(temp, CJKMonthSuff, TokenType.SEP_MonthSuff, 0); 1966InsertHash(temp, KoreanMonthSuff, TokenType.SEP_MonthSuff, 0); 1967InsertHash(temp, CJKDaySuff, TokenType.SEP_DaySuff, 0); 1968InsertHash(temp, KoreanDaySuff, TokenType.SEP_DaySuff, 0); 1970InsertHash(temp, CJKHourSuff, TokenType.SEP_HourSuff, 0); 1971InsertHash(temp, ChineseHourSuff, TokenType.SEP_HourSuff, 0); 1972InsertHash(temp, CJKMinuteSuff, TokenType.SEP_MinuteSuff, 0); 1973InsertHash(temp, CJKSecondSuff, TokenType.SEP_SecondSuff, 0); 1979InsertHash(temp, JapaneseEraStart, TokenType.YearNumberToken, 1); 1980InsertHash(temp, "(", TokenType.IgnorableSymbol, 0); 1981InsertHash(temp, ")", TokenType.IgnorableSymbol, 0); 1988InsertHash(temp, KoreanHourSuff, TokenType.SEP_HourSuff, 0); 1989InsertHash(temp, KoreanMinuteSuff, TokenType.SEP_MinuteSuff, 0); 1990InsertHash(temp, KoreanSecondSuff, TokenType.SEP_SecondSuff, 0); 1996InsertHash(temp, dateSeparatorOrTimeZoneOffset, TokenType.IgnorableSymbol, 0); 2000InsertHash(temp, dateSeparatorOrTimeZoneOffset, TokenType.SEP_DateOrOffset, 0); 2032InsertHash(temp, symbol, TokenType.IgnorableSymbol, 0); 2040InsertHash(temp, dateWords[i], TokenType.DateWordToken, 0); 2045InsertHash(temp, IgnorablePeriod + dateWords[i], TokenType.DateWordToken, 0); 2055InsertHash(temp, DateSeparator, TokenType.SEP_Date, 0); 2063InsertHash(temp, GetAbbreviatedMonthName(i), TokenType.MonthToken, i); 2073InsertHash(temp, genitiveMonthNames[i - 1], TokenType.MonthToken, i); 2074InsertHash(temp, abbreviatedGenitiveMonthNames[i - 1], TokenType.MonthToken, i); 2083InsertHash(temp, str, TokenType.MonthToken, i); 2091InsertHash(temp, str, TokenType.DayOfWeekToken, i); 2094InsertHash(temp, str, TokenType.DayOfWeekToken, i); 2100InsertHash(temp, GetEraName(i), TokenType.EraToken, i); 2101InsertHash(temp, GetAbbreviatedEraName(i), TokenType.EraToken, i); 2112InsertHash(temp, specialDayOfWeek, TokenType.DayOfWeekToken, i); 2121InsertHash(temp, jaDtfi.GetEraName(i), TokenType.JapaneseEraToken, i); 2122InsertHash(temp, jaDtfi.GetAbbreviatedEraName(i), TokenType.JapaneseEraToken, i); 2124InsertHash(temp, jaDtfi.AbbreviatedEnglishEraNames[i - 1], TokenType.JapaneseEraToken, i); 2136InsertHash(temp, twDtfi.GetEraName(i), TokenType.TEraToken, i); 2142InsertHash(temp, InvariantInfo.AMDesignator, TokenType.SEP_Am | TokenType.Am, 0); 2143InsertHash(temp, InvariantInfo.PMDesignator, TokenType.SEP_Pm | TokenType.Pm, 1); 2152InsertHash(temp, str, TokenType.MonthToken, i); 2154InsertHash(temp, str, TokenType.MonthToken, i); 2161InsertHash(temp, str, TokenType.DayOfWeekToken, i); 2164InsertHash(temp, str, TokenType.DayOfWeekToken, i); 2170InsertHash(temp, AbbreviatedEnglishEraNames[i], TokenType.EraToken, i + 1); 2173InsertHash(temp, LocalTimeMark, TokenType.SEP_LocalTimeMark, 0); 2174InsertHash(temp, GMTName, TokenType.TimeZoneToken, 0); 2175InsertHash(temp, ZuluName, TokenType.TimeZoneToken, 0); 2177InsertHash(temp, invariantDateSeparator, TokenType.SEP_Date, 0); 2178InsertHash(temp, invariantTimeSeparator, TokenType.SEP_Time, 0); 2198InsertHash(temp, string.Concat(str, monthPostfix), TokenType.MonthToken, i); 2202InsertHash(temp, str, TokenType.MonthToken, i); 2206InsertHash(temp, str, TokenType.MonthToken, i); 2293internal bool Tokenize(TokenType TokenMask, out TokenType tokenType, out int tokenValue, 2296tokenType = TokenType.UnknownToken; 2307if (!GlobalizationMode.Invariant && IsHebrewChar(ch) && TokenMask == TokenType.RegularTokenMask) 2313tokenType = TokenType.UnknownToken; 2318tokenType = TokenType.HebrewNumber; 2368else if ((value.tokenType == TokenType.MonthToken && HasSpacesInMonthNames) || 2369(value.tokenType == TokenType.DayOfWeekToken && HasSpacesInDayNames)) 2390private void InsertAtCurrentHashNode(TokenHashValue[] hashTable, string str, char ch, TokenType tokenType, int tokenValue, int pos, int hashcode, int hashProbe) 2421private void InsertHash(TokenHashValue[] hashTable, string str, TokenType tokenType, int tokenValue) 2483if (((nCurrentTokenTypeInHash & (int)TokenType.RegularTokenMask) == 0) && ((nTokenType & (int)TokenType.RegularTokenMask) != 0) || 2484((nCurrentTokenTypeInHash & (int)TokenType.SeparatorTokenMask) == 0) && ((nTokenType & (int)TokenType.SeparatorTokenMask) != 0)) 2519internal TokenType tokenType; 2522internal TokenHashValue(string tokenString, TokenType tokenType, int tokenValue)

src\runtime\src\libraries\System.Private.CoreLib\src\System\Globalization\DateTimeParse.cs (84)

620TokenType sep; 623str.GetRegularToken(out TokenType tokenType, out int tokenValue, dtfi); 635case TokenType.NumberToken: 636case TokenType.YearNumberToken: 683if (tokenType == TokenType.YearNumberToken) 695case TokenType.SEP_End: 698case TokenType.SEP_Am: 699case TokenType.SEP_Pm: 702raw.timeMark = (sep == TokenType.SEP_Am ? TM.AM : TM.PM); 711case TokenType.SEP_Space: 714case TokenType.SEP_Date: 717case TokenType.SEP_Time: 729case TokenType.SEP_DateOrOffset: 743case TokenType.SEP_YearSuff: 744case TokenType.SEP_MonthSuff: 745case TokenType.SEP_DaySuff: 749case TokenType.SEP_HourSuff: 750case TokenType.SEP_MinuteSuff: 751case TokenType.SEP_SecondSuff: 777case TokenType.SEP_End: 781case TokenType.SEP_Am: 782case TokenType.SEP_Pm: 785raw.timeMark = (sep == TokenType.SEP_Am ? TM.AM : TM.PM); 812case TokenType.SEP_Space: 816case TokenType.SEP_Date: 820case TokenType.SEP_DateOrOffset: 835case TokenType.SEP_Time: 847case TokenType.SEP_YearSuff: 861case TokenType.SEP_MonthSuff: 862case TokenType.SEP_DaySuff: 866case TokenType.SEP_HourSuff: 867case TokenType.SEP_MinuteSuff: 868case TokenType.SEP_SecondSuff: 872case TokenType.SEP_LocalTimeMark: 883case TokenType.HebrewNumber: 897case TokenType.SEP_End: 900case TokenType.SEP_Space: 903case TokenType.SEP_DateOrOffset: 948case TokenType.SEP_End: 951case TokenType.SEP_Space: 952case TokenType.SEP_Date: 955case TokenType.SEP_DateOrOffset: 977case TokenType.DayOfWeekToken: 993case TokenType.MonthToken: 1001case TokenType.SEP_End: 1004case TokenType.SEP_Space: 1007case TokenType.SEP_Date: 1010case TokenType.SEP_Time: 1022case TokenType.SEP_DateOrOffset: 1051case TokenType.EraToken: 1064case TokenType.JapaneseEraToken: 1086case TokenType.TEraToken: 1107case TokenType.TimeZoneToken: 1125case TokenType.EndOfString: 1128case TokenType.DateWordToken: 1129case TokenType.IgnorableSymbol: 1132case TokenType.Am: 1133case TokenType.Pm: 1145case TokenType.UnknownToken: 2187case TokenType.SEP_YearSuff: 2195case TokenType.SEP_MonthSuff: 2203case TokenType.SEP_DaySuff: 2211case TokenType.SEP_HourSuff: 2219case TokenType.SEP_MinuteSuff: 2227case TokenType.SEP_SecondSuff: 2524dtok.suffix = TokenType.SEP_Unk; 2566if (dtok.suffix != TokenType.SEP_Unk) 2575dtok.suffix = TokenType.SEP_Unk; // Reset suffix to SEP_Unk; 5590internal void GetRegularToken(out TokenType tokenType, out int tokenValue, DateTimeFormatInfo dtfi) 5595tokenType = TokenType.EndOfString; 5625tokenType = TokenType.NumberToken; 5630tokenType = TokenType.NumberToken; 5635tokenType = TokenType.YearNumberToken; 5646if (dtfi.Tokenize(TokenType.RegularTokenMask, out TokenType tempType, out int tempValue, ref this)) 5673tokenType = TokenType.EndOfString; 5677dtfi.Tokenize(TokenType.RegularTokenMask, out tokenType, out tokenValue, ref this); 5681internal TokenType GetSeparatorToken(DateTimeFormatInfo dtfi, out int indexBeforeSeparator, out char charBeforeSeparator) 5685TokenType tokenType; 5689return TokenType.SEP_End; 5694bool found = dtfi.Tokenize(TokenType.SeparatorTokenMask, out tokenType, out _, ref this); 5697tokenType = TokenType.SEP_Space; 5704tokenType = TokenType.SEP_Space; 6126internal TokenType suffix; // Store the CJK Year/Month/Day suffix (if any)