File: Expression\Tokens.cs
Web Access
Project: src\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj (Microsoft.ML.Transforms)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
 
using System;
using System.Globalization;
using System.IO;
using Microsoft.ML.Runtime;
 
namespace Microsoft.ML.Transforms
{
    internal struct TextSpan
    {
        public readonly int Min;
        public readonly int Lim;
 
        public TextSpan(int ichMin, int ichLim)
        {
            Contracts.Assert(0 <= ichMin && ichMin <= ichLim);
            Min = ichMin;
            Lim = ichLim;
        }
 
        public override string ToString()
        {
            return string.Format(CultureInfo.InvariantCulture, "({0},{1})", Min, Lim);
        }
    }
 
    internal abstract class Token
    {
        public readonly TokKind Kind;
        public readonly TokKind KindContext;
        public readonly TextSpan Span;
 
        protected Token(TextSpan span, TokKind tid)
        {
            Span = span;
            Kind = tid;
            KindContext = tid;
        }
 
        protected Token(TextSpan span, TokKind tid, TokKind tidContext)
        {
            // Currently the only contextual variability is that an identifier might double as a keyword.
            Contracts.Assert(tidContext == tid || tid == TokKind.Ident);
            Span = span;
            Kind = tid;
            KindContext = tidContext;
        }
 
        public T As<T>() where T : Token
        {
            Contracts.Assert(this is T);
            return (T)this;
        }
 
        public override string ToString()
        {
            return Kind.ToString();
        }
    }
 
    // Keyword/punctuation token
    internal sealed class KeyToken : Token
    {
        public static KeyToken Create(TextSpan span, TokKind tid)
        {
            return new KeyToken(span, tid);
        }
 
        public static Token CreateKeyWord(TextSpan span, string str, TokKind tid, bool isContextKeyWord)
        {
            if (isContextKeyWord)
                return new IdentToken(span, str, tid);
            return new KeyToken(span, tid);
        }
 
        private KeyToken(TextSpan span, TokKind tid)
            : base(span, tid)
        {
        }
    }
 
    internal sealed class IdentToken : Token
    {
        public readonly string Value;
        public IdentToken(TextSpan span, string val) : base(span, TokKind.Ident) { Value = val; }
        public IdentToken(TextSpan span, string val, TokKind tidContext) : base(span, TokKind.Ident, tidContext) { Value = val; }
        public override string ToString()
        {
            if (KindContext != TokKind.Ident)
                return "ContextKeyword: " + Value;
            return "Ident: " + Value;
        }
    }
 
    [Flags]
    internal enum IntLitKind : byte
    {
        None = 0x00,
        Uns = 0x01,
        Lng = 0x02,
        Hex = 0x04,
 
        UnsLng = Uns | Lng,
    }
 
    internal abstract class NumLitToken : Token
    {
        protected NumLitToken(TextSpan span, TokKind tid)
            : base(span, tid)
        {
        }
    }
 
    internal sealed class IntLitToken : NumLitToken
    {
        public readonly ulong Value;
        public readonly IntLitKind IntKind; // The kind specified by suffixes.
 
        public IntLitToken(TextSpan span, ulong val, IntLitKind ilk)
            : base(span, TokKind.IntLit)
        {
            Value = val;
            IntKind = ilk;
        }
 
        public override string ToString()
        {
            string suff1 = (IntKind & IntLitKind.Uns) != 0 ? "U" : "";
            string suff2 = (IntKind & IntLitKind.Lng) != 0 ? "L" : "";
            string fmt;
 
            if (IsHex)
                fmt = Value < 0x0000000100000000 ? "0x{0:X8}{1}{2}" : "0x{0:X16}{1}{2}";
            else
                fmt = "{0}{1}{2}";
            return string.Format(fmt, Value, suff1, suff2);
        }
 
        public bool IsHex { get { return (IntKind & IntLitKind.Hex) != 0; } }
    }
 
    internal sealed class FltLitToken : NumLitToken
    {
        public readonly float Value;
        public FltLitToken(TextSpan span, float val) : base(span, TokKind.FltLit) { Value = val; }
        public override string ToString()
        {
            if (float.IsPositiveInfinity(Value))
                return "1e1000f";
            return Value.ToString("R") + "f";
        }
    }
 
    internal sealed class DblLitToken : NumLitToken
    {
        public readonly double Value;
        // Whether this literal has an explicit d/D suffix.
        public readonly bool HasSuffix;
 
        public DblLitToken(TextSpan span, double val, bool hasSuf)
            : base(span, TokKind.DblLit)
        {
            Value = val;
            HasSuffix = hasSuf;
        }
 
        public override string ToString()
        {
            string res;
            if (double.IsPositiveInfinity(Value))
                res = "1e1000";
            else
                res = Value.ToString("R");
            if (HasSuffix)
                res += "d";
            else if (AllDigits(res))
                res += ".0";
            return res;
        }
 
        private static bool AllDigits(string s)
        {
            Contracts.AssertNonEmpty(s);
            for (int i = 0; i < s.Length; i++)
            {
                if ((uint)(s[i] - '0') > 9)
                    return false;
            }
            return true;
        }
    }
 
    internal sealed class CharLitToken : Token
    {
        public readonly char Value;
        public CharLitToken(TextSpan span, char val) : base(span, TokKind.CharLit) { Value = val; }
        public override string ToString()
        {
            if (Value < ' ' || Value >= 0x7F)
                return string.Format(@"'\u{0:X4}'", (int)Value);
            if (Value == '\'')
                return @"'\''";
            return string.Format("'{0}'", Value);
        }
    }
 
    internal sealed class StrLitToken : Token
    {
        public readonly string Value;
        public StrLitToken(TextSpan span, string val) : base(span, TokKind.StrLit) { Value = val; }
        public override string ToString()
        {
            StringWriter wrt = new StringWriter();
            Write(wrt, Value);
            return wrt.ToString();
        }
 
        public static void Write(TextWriter wrt, string str)
        {
            Contracts.AssertValue(str);
 
            wrt.Write('"');
            int ich = 0;
            for (; ich < str.Length; ich++)
            {
                char ch = str[ich];
                if (ch < ' ' || ch >= 0x7F)
                    wrt.Write(@"\u{0:X4}", (int)ch);
                else
                {
                    if (ch == '\\' || ch == '"')
                        wrt.Write('\\');
                    wrt.Write(ch);
                }
            }
            wrt.Write('"');
        }
    }
 
    internal sealed class CommentToken : Token
    {
        public readonly string Text;
        public readonly int NewLineCount;
        public CommentToken(TextSpan span, string val, int lines)
            : base(span, TokKind.Comment)
        {
            Text = val;
            NewLineCount = lines;
        }
        public override string ToString() { return "Comment: " + Text; }
    }
 
    internal sealed class NewLineToken : Token
    {
        public readonly bool Nested;
        public NewLineToken(TextSpan span, bool fNested)
            : base(span, TokKind.NewLine)
        {
            Nested = fNested;
        }
        public override string ToString() { return Nested ? "NewLine<Nested>" : "NewLine"; }
    }
 
    internal sealed class ErrorToken : Token
    {
        public readonly ErrId Id;
        public readonly object[] Args;
 
        public ErrorToken(TextSpan span, ErrId eid, params object[] args)
            : base(span, TokKind.Error)
        {
            Id = eid;
            Args = args;
        }
 
        public override string ToString()
        {
            return Id.GetMsg(Args);
        }
    }
 
    internal sealed class EofToken : Token
    {
        public EofToken(TextSpan span)
            : base(span, TokKind.Eof)
        {
        }
    }
}