`OsfV1Tokenizer.java`

    // FILE. . . . . /home/hak/hlt/src/hlt/osfv1/apps/v1/sources/OsfV0Tokenizer.java
    // EDIT BY . . . Hassan Ait-Kaci
    // ON MACHINE. . Hak-Laptop
    // STARTED ON. . Sun Sep 08 18:31:03 2013

This is the tokenizer for the OSF V1 language grammar. It defines the lexical units used in the grammar rules OsfV1Rules.grm and declared in OsfV1Declarations.grm.
Author: |  | Hassan Aït-Kaci
Copyright: |  | © by the author
Version: |  | Last modified on Mon Sep 16 20:14:40 2013 by hak

    package hlt.osf.v1;

    import java.io.IOException;
    import java.io.Reader;
    import java.io.FileReader;
    import java.io.StringReader;
    import java.io.InputStreamReader;
    import java.io.IOException;
    import java.io.FileNotFoundException;

    import hlt.language.io.CircularInclusionException;
    import hlt.language.io.IncludeReader;
    import hlt.language.io.StreamTokenizer;
    import hlt.language.io.IO;

    import hlt.language.tools.Misc;
    // import hlt.language.tools.Debug;

    import hlt.language.syntax.*;

    public class OsfV1Tokenizer extends AbstractTokenizer
    {
      int lineno;
      IncludeReader reader;
      StreamTokenizer st;
      boolean isInteractive;

      public final void setInteractive ()
        {
          setReader(new InputStreamReader(System.in));
          isInteractive = true;
          prompt();
        }

      public final int lineNumber()
        {
          return lineno;
        }

      public final void setReader (Reader rd)
        {
          reader = rd instanceof IncludeReader ? (IncludeReader)rd : new IncludeReader(rd);
          st = new StreamTokenizer(reader);
          st.ordinaryChars(".(){}[]<>;,?:#%^&|!=-~`*/\\");
          st.setType('.',st.NUMERIC);
          // st.setType('.',st.ORDINARY);
          st.parseDigitsOnly();
          st.quoteChar('\'');
          st.quoteChar('"');
          st.wordChars("-+_@");

          // Debug.setDebugFlag(true);
        }

      public final Reader getReader ()
        {
          return reader;
        }

      final void setFile (String file)
        {
          reader.setFile(file);
        }

      public final void include (String file) throws FileNotFoundException, CircularInclusionException
        {
          reader.include(file);
        }

      public final void setIsInteractive (boolean flag)
        {
          isInteractive = flag;
        }

      private final boolean promptIsNeeded ()
        {
          boolean isWaiting = isInteractive;

          try
            {
    	  isWaiting &= (!reader.ready() && reader.depth() == 0);
    	            // || (reader.ready() && reader.depth() == 1 && reader.reachedEndOfInclusion());

    	  // if (Debug.flagIsOn())
    	  //   {
    	  //     System.out.println("Reading from "+(reader.getFile() == null ? "console" : reader.getFile()));
    	  //     System.out.println("Reader ready? = "+reader.ready());
    	  //     System.out.println("Reader depth = "+reader.depth());
    	  //     System.out.println("Latest char = "+Misc.pform(reader.latestChar()));
    	  //   }
            }
          catch (IOException e)
            {
              e.printStackTrace();
            }

          return isWaiting || reader.reachedEndOfInclusion();
        }

      public final void prompt ()
        {
          if (promptIsNeeded())
    	{
    	  // if (OsfV1Parser.isTiming)
    	  //   {
    	  //     OsfV1Parser.time = System.currentTimeMillis() - OsfV1Parser.time;
    	  //     OsfV1Parser.displayLine("*** Processing time = "+OsfV1Parser.time+" ms");
    	  //   }
    	  System.out.print(promptString());
    	}
        }

      public final String promptString ()
        {
          return Misc.numberString(lineno+1,3,'*') + " " + promptLabel + "> ";
        }

      String promptLabel = "OSF V1.0";

      public final void setPrompt (String p)
        {
          promptLabel = p;
        }

      final ParseNode locate (ParseNode node)
        {
          return ((ParseNode)node
    	      .setStart(st.tokenStart())
    	      .setEnd(st.tokenEnd())).setFile(reader.getFile());
        }

      /* ************************************************************************ */

      final ParseNode integer ()
        {
          return locate(GenericParser.numberToken("INTEGER",(int)st.nval));
          // return locate(GenericParser.symbolToken("IDENTIFIER",Integer.toString((int)st.nval)));
        }

      final ParseNode character ()
        {
          return locate(GenericParser.symbolToken("CHAR",st.sval));
        }

      final ParseNode floatingPoint ()
        {
          return locate(GenericParser.numberToken("FLOAT",st.nval));
        }

      final ParseNode string ()
        {
          return locate(GenericParser.symbolToken("STRING",st.sval));
        }

      final ParseNode identifier ()
        {
          return locate(GenericParser.symbolToken("IDENTIFIER",st.sval));
        }

      final ParseNode truthvalue ()
        {
          return locate(GenericParser.symbolToken("BOOLEAN",st.sval));
        }

      final ParseNode pragma ()
        {
          return locate(GenericParser.symbolToken("PRAGMA",st.sval.intern()));
        }

      final ParseNode nameTag ()
        {
          return locate(GenericParser.symbolToken("TAG",("#"+st.sval).intern()));
        }

      final ParseNode numberTag ()
        {
          return locate(GenericParser.symbolToken("TAG",("#"+(int)st.nval).intern()));
        }

      final ParseNode top ()
        {
          return locate(GenericParser.symbolToken("TOP","@"));
        }

      final ParseNode bottom ()
        {
          return locate(GenericParser.symbolToken("BOTTOM","{}"));
        }

      final ParseNode isa ()
        {
          return locate(GenericParser.symbolToken("ISA","<|"));
        }

      final ParseNode arrow ()
        {
          return locate(GenericParser.symbolToken("ARROW","=>"));
        }

      final ParseNode and ()
        {
          return locate(GenericParser.symbolToken("AND","&"));
        }

      final ParseNode or ()
        {
          return locate(GenericParser.symbolToken("OR","|"));
        }

      final ParseNode not ()
        {
          return locate(GenericParser.symbolToken("NOT","!"));
        }

      final ParseNode butnot ()
        {
          return locate(GenericParser.symbolToken("BUTNOT","\\"));
        }

      final ParseNode eos ()
        {
          return locate(GenericParser.symbolToken("EOS","."));
        }

      final ParseNode literal (String s)
        {
          return locate(GenericParser.literalToken(s));
        }

      final ParseNode literal (int c)
        {
          return locate(GenericParser.literalToken(String.valueOf((char)c)));
        }

      /* ************************************************************************ */

      public final ParseNode nextToken () throws IOException
        {
          ParseNode t = null;
          int tk = st.nextToken();
          int nt = 0;
          lineno = st.lineno();

          // if (Debug.flagIsOn())
          // 	{
          // 	  System.out.println("Stream tokenizer's current state: "+st);
          // 	}

          switch (tk)
            {
            case StreamTokenizer.TT_SPECIAL:
              return nextToken();
            case IO.EOI:
    	  System.out.println("*** END OF INPUT");
    	  return nextToken();
            case StreamTokenizer.TT_EOF:
    	  // if (Debug.flagIsOn())
    	  //   System.out.println("READ END OF FILE");
              t = locate(GenericParser.eoi());
              break;
            case StreamTokenizer.TT_WORD:
    	  if (st.sval == "true" || st.sval == "false")
    	    t = truthvalue();
    	  else
    	    t = identifier();
    	  break;
            case '%':
    	  if ((nt = st.nextToken()) == StreamTokenizer.TT_WORD)
    	    t = pragma();
    	  else
    	    {
    	      st.pushBack();
    	      t = literal("%");
    	    }
    	  break;
            case '#':
    	  if ((nt = st.nextToken()) == StreamTokenizer.TT_WORD)
    	    t = nameTag();
    	  else
    	    if (nt == StreamTokenizer.TT_NUMBER && st.isInteger)
    	      t = numberTag();
    	      else
    		{
    		  st.pushBack();
    		  t = literal("#");
    		}
    	  break;
            case '"':
              t = string();
              break;
            case '\'':    // NB: this does NOT check the contents between single quotes
              t = character();
              break;
    	case '@':
    	  t = top();
    	  break;
    	case '{':
              if (st.peek() == '}')
                {
    	      t = bottom();
                  st.skipChar(true);
    	    }
    	  else
    	    t = literal("{");
              break;
            case '<':
              if (st.peek() == '|')
                {
    	      t = isa();
                  st.skipChar(true);
    	    }
    	  else
    	    t = literal("<");
              break;
            case '=':
              if (st.peek() == '>')
                {
    	      t = arrow();
                  st.skipChar(true);
    	    }
    	  else
    	    t = literal("=");
              break;
            case '&':
    	  t = and();
              break;
            case '|':
    	  t = or();
              break;
            case '!':
    	  t = not();
              break;
            case '\\':
    	  t = butnot();
              break;
            case StreamTokenizer.TT_NUMBER:
              if (st.isInteger)
                t = integer();
              else
                t = floatingPoint();
              break;
            case '.':
    	  t = eos();
              break;
    	default:
    	  t = literal(tk);
            }
          // System.out.println("Read Token: "+t.nodeInfo());
          return t;
        }
    }

This file was generated on Thu Oct 03 16:39:07 CEST 2013 from file `OsfV1Tokenizer.java`
by the `hlt.language.tools.Hilite` Java tool written by Hassan Aït-Kaci