compiler2/asn1/TokenBuf.cc

   1 /******************************************************************************
   2  * Copyright (c) 2000-2016 Ericsson Telecom AB
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Eclipse Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Contributors:
   9  *   Balasko, Jeno
  10  *   Cserveni, Akos
  11  *   Delic, Adam
  12  *   Feher, Csaba
  13  *   Forstner, Matyas
  14  *   Kovacs, Ferenc
  15  *   Raduly, Csaba
  16  *   Szabo, Janos Zoltan – initial implementation
  17  *
  18  ******************************************************************************/
  19 #include "TokenBuf.hh"
  20 #include "../AST.hh"
  21 #include "Block.hh"
  22
  23 extern void asn1_yyerror(const char *s);
  24 #define yyerror asn1_yyerror
  25 extern int asn1_yylex();
  26 #define yylex asn1_yylex
  27 extern const char *asn1_infile;
  28 extern int asn1_yylineno;
  29 #define yylineno asn1_yylineno
  30 extern int asn1_plineno;
  31 #define plineno asn1_plineno
  32 extern YYLTYPE asn1_yylloc;
  33 #define yylloc asn1_yylloc
  34
  35 namespace Asn {
  36
  37   using namespace Common;
  38
  39   // =================================
  40   // ===== Token
  41   // =================================
  42
  43   Token::Token(const Token& p)
  44     : Node(p), Location(p), token(p.token)
  45   {
  46     switch (p.token) {
  47     case TOK_UpperIdentifier:
  48     case TOK_LowerIdentifier:
  49     case TOK_ampUpperIdentifier:
  50     case TOK_ampLowerIdentifier:
  51       semval.id = p.semval.id->clone();
  52       break;
  53     case TOK_Number:
  54       semval.i = new int_val_t(*p.semval.i);
  55       break;
  56     case TOK_CString:
  57       semval.str = new string(*p.semval.str);
  58       break;
  59     case TOK_RealNumber:
  60     case TOK_BString:
  61     case TOK_HString:
  62       semval.value = p.semval.value->clone();
  63       break;
  64     case TOK_Block:
  65       semval.block = p.semval.block->clone();
  66     default:
  67       break;
  68     }
  69   }
  70
  71   bool Token::has_semval(int p_token)
  72   {
  73     switch (p_token) {
  74     case TOK_UpperIdentifier:
  75     case TOK_LowerIdentifier:
  76     case TOK_ampUpperIdentifier:
  77     case TOK_ampLowerIdentifier:
  78     case TOK_Number:
  79     case TOK_CString:
  80     case TOK_RealNumber:
  81     case TOK_BString:
  82     case TOK_HString:
  83     case TOK_Block:
  84       return true;
  85     default:
  86       return false;
  87     }
  88   }
  89
  90   Token::Token(int p_token, const Location& p_loc)
  91     : Node(), Location(p_loc), token(p_token)
  92   {
  93     if (has_semval(p_token)) FATAL_ERROR("Token::Token()");
  94   }
  95
  96   Token::Token(int p_token, const YYSTYPE& p_semval, const Location& p_loc)
  97     : Node(), Location(p_loc), token(p_token)
  98   {
  99     switch (p_token) {
 100     case TOK_UpperIdentifier:
 101     case TOK_LowerIdentifier:
 102     case TOK_ampUpperIdentifier:
 103     case TOK_ampLowerIdentifier:
 104       semval.id = p_semval.id;
 105       break;
 106     case TOK_Number:
 107       semval.i = p_semval.i;
 108       break;
 109     case TOK_CString:
 110       semval.str = p_semval.str;
 111       break;
 112     case TOK_RealNumber:
 113     case TOK_BString:
 114     case TOK_HString:
 115       semval.value = p_semval.value;
 116       break;
 117     case TOK_Block:
 118       semval.block = p_semval.block;
 119     default:
 120       break;
 121     }
 122   }
 123
 124   Token::~Token()
 125   {
 126     switch(token) {
 127     case TOK_UpperIdentifier:
 128     case TOK_LowerIdentifier:
 129     case TOK_ampUpperIdentifier:
 130     case TOK_ampLowerIdentifier:
 131       delete semval.id;
 132       break;
 133     case TOK_CString:
 134       delete semval.str;
 135       break;
 136     case TOK_RealNumber:
 137     case TOK_BString:
 138     case TOK_HString:
 139       delete semval.value;
 140       break;
 141     case TOK_Block:
 142       delete semval.block;
 143       break;
 144     case TOK_Number:
 145       delete semval.i;
 146       break;
 147     default:
 148       break;
 149     } // switch
 150   }
 151
 152   Token *Token::clone() const
 153   {
 154     return new Token(*this);
 155   }
 156
 157   void Token::set_token(int new_token)
 158   {
 159     if (new_token != token) {
 160       if (has_semval(token) || has_semval(new_token))
 161         FATAL_ERROR("Token::set_token()");
 162       token = new_token;
 163     }
 164   }
 165
 166   void Token::steal_semval(YYSTYPE& p_semval)
 167   {
 168     switch (token) {
 169     case TOK_UpperIdentifier:
 170     case TOK_LowerIdentifier:
 171     case TOK_ampUpperIdentifier:
 172     case TOK_ampLowerIdentifier:
 173       p_semval.id = semval.id;
 174       break;
 175     case TOK_Number:
 176       p_semval.i = semval.i;
 177       break;
 178     case TOK_CString:
 179       p_semval.str = semval.str;
 180       break;
 181     case TOK_RealNumber:
 182     case TOK_BString:
 183     case TOK_HString:
 184       p_semval.value = semval.value;
 185       break;
 186     case TOK_Block:
 187       p_semval.block = semval.block;
 188     default:
 189       break;
 190     }
 191     token = '\0';
 192   }
 193
 194   void Token::set_loc_info() const
 195   {
 196     asn1_infile = get_filename();
 197     if(get_lineno() > 0) plineno = get_lineno();
 198   }
 199
 200   bool Token::is_literal_id() const
 201   {
 202     if(token==TOK_UpperIdentifier
 203        && semval.id->isvalid_asn_word())
 204       return true;
 205     else return false;
 206   }
 207
 208   bool Token::is_literal_kw() const
 209   {
 210     switch(token) {
 211     case ',':
 212     case KW_ABSENT:
 213     case KW_ALL:
 214     case KW_ANY:
 215     case KW_APPLICATION:
 216     case KW_AUTOMATIC:
 217     case KW_BEGIN:
 218     case KW_BY:
 219     case KW_CLASS:
 220     case KW_COMPONENT:
 221     case KW_COMPONENTS:
 222     case KW_CONSTRAINED:
 223     case KW_CONTAINING:
 224     case KW_DEFAULT:
 225     case KW_DEFINED:
 226     case KW_DEFINITIONS:
 227     case KW_ENCODED:
 228     case KW_EXCEPT:
 229     case KW_EXPLICIT:
 230     case KW_EXPORTS:
 231     case KW_EXTENSIBILITY:
 232     case KW_FROM:
 233     case KW_IDENTIFIER:
 234     case KW_IMPLICIT:
 235     case KW_IMPLIED:
 236     case KW_IMPORTS:
 237     case KW_INCLUDES:
 238     case KW_MAX:
 239     case KW_MIN:
 240     case KW_OF:
 241     case KW_OPTIONAL:
 242     case KW_PATTERN:
 243     case KW_PDV:
 244     case KW_PRESENT:
 245     case KW_PRIVATE:
 246     case KW_SIZE:
 247     case KW_STRING:
 248     case KW_SYNTAX:
 249     case KW_TAGS:
 250     case KW_UNIQUE:
 251     case KW_UNIVERSAL:
 252     case KW_WITH:
 253       return true;
 254     default:
 255       return false;
 256     } // switch token
 257   }
 258
 259   bool Token::is_ampId() const
 260   {
 261     if(token==TOK_ampUpperIdentifier
 262        || token==TOK_ampLowerIdentifier)
 263       return true;
 264     else return false;
 265   }
 266
 267   bool Token::is_id() const
 268   {
 269     switch(token) {
 270     case TOK_UpperIdentifier:
 271     case TOK_LowerIdentifier:
 272     case TOK_ampUpperIdentifier:
 273     case TOK_ampLowerIdentifier:
 274       return true;
 275     default:
 276       return false;
 277     } // switch
 278   }
 279
 280   const char* Token::get_token_name(int p_token)
 281   {
 282     switch(p_token) {
 283     case '\0': return "<end of file or statement>";
 284     case TOK_Assignment: return "::=";
 285     case TOK_RangeSeparator: return "..";
 286     case TOK_Ellipsis: return "...";
 287     case TOK_LeftVersionBrackets: return "[[";
 288     case TOK_RightVersionBrackets: return "]]";
 289     case '{': return "{";
 290     case '}': return "}";
 291     case '(': return "(";
 292     case ')': return ")";
 293     case '[': return "[";
 294     case ']': return "]";
 295     case ',': return ",";
 296     case '.': return ".";
 297     case '-': return "-";
 298     case ':': return ":";
 299     case ';': return ";";
 300     case '@': return "@";
 301     case '|': return "|";
 302     case '!': return "!";
 303     case '^': return "^";
 304     case '<': return "<";
 305     case '>': return ">";
 306     case '\'': return "'";
 307     case '"': return "\"";
 308     case KW_ABSENT: return "ABSENT";
 309     case KW_ALL: return "ALL";
 310     case KW_ANY: return "ANY";
 311     case KW_APPLICATION: return "APPLICATION";
 312     case KW_AUTOMATIC: return "AUTOMATIC";
 313     case KW_BEGIN: return "BEGIN";
 314     case KW_BIT: return "BIT";
 315     case KW_BMPString: return "BMPString";
 316     case KW_BOOLEAN: return "BOOLEAN";
 317     case KW_BY: return "BY";
 318     case KW_CHARACTER: return "CHARACTER";
 319     case KW_CHOICE: return "CHOICE";
 320     case KW_CLASS: return "CLASS";
 321     case KW_COMPONENT: return "COMPONENT";
 322     case KW_COMPONENTS: return "COMPONENTS";
 323     case KW_CONSTRAINED: return "CONSTRAINED";
 324     case KW_CONTAINING: return "CONTAINING";
 325     case KW_DEFAULT: return "DEFAULT";
 326     case KW_DEFINED: return "DEFINED";
 327     case KW_DEFINITIONS: return "DEFINITIONS";
 328     case KW_EMBEDDED: return "EMBEDDED";
 329     case KW_ENCODED: return "ENCODED";
 330     case KW_END: return "END";
 331     case KW_ENUMERATED: return "ENUMERATED";
 332     case KW_EXCEPT: return "EXCEPT";
 333     case KW_EXPLICIT: return "EXPLICIT";
 334     case KW_EXPORTS: return "EXPORTS";
 335     case KW_EXTENSIBILITY: return "EXTENSIBILITY";
 336     case KW_EXTERNAL: return "EXTERNAL";
 337     case KW_FALSE: return "FALSE";
 338     case KW_GeneralizedTime: return "GeneralizedTime";
 339     case KW_GeneralString: return "GeneralString";
 340     case KW_GraphicString: return "GraphicString";
 341     case KW_IA5String: return "IA5String";
 342     case KW_FROM: return "FROM";
 343     case KW_IDENTIFIER: return "IDENTIFIER";
 344     case KW_IMPLICIT: return "IMPLICIT";
 345     case KW_IMPLIED: return "IMPLIED";
 346     case KW_IMPORTS: return "IMPORTS";
 347     case KW_INCLUDES: return "INCLUDES";
 348     case KW_INSTANCE: return "INSTANCE";
 349     case KW_INTEGER: return "INTEGER";
 350     case KW_INTERSECTION: return "INTERSECTION";
 351     case KW_ISO646String: return "ISO646String";
 352     case KW_MAX: return "MAX";
 353     case KW_MIN: return "MIN";
 354     case KW_MINUS_INFINITY: return "MINUS-INFINITY";
 355     case KW_NULL: return "NULL";
 356     case KW_NumericString: return "NumericString";
 357     case KW_OBJECT: return "OBJECT";
 358     case KW_ObjectDescriptor: return "ObjectDescriptor";
 359     case KW_OCTET: return "OCTET";
 360     case KW_OF: return "OF";
 361     case KW_OPTIONAL: return "OPTIONAL";
 362     case KW_PATTERN: return "PATTERN";
 363     case KW_PDV: return "PDV";
 364     case KW_PLUS_INFINITY: return "PLUS-INFINITY";
 365     case KW_PRESENT: return "PRESENT";
 366     case KW_PrintableString: return "PrintableString";
 367     case KW_PRIVATE: return "PRIVATE";
 368     case KW_REAL: return "REAL";
 369     case KW_RELATIVE_OID: return "RELATIVE-OID";
 370     case KW_SEQUENCE: return "SEQUENCE";
 371     case KW_SET: return "SET";
 372     case KW_SIZE: return "SIZE";
 373     case KW_STRING: return "STRING";
 374     case KW_SYNTAX: return "SYNTAX";
 375     case KW_T61String: return "T61String";
 376     case KW_TAGS: return "TAGS";
 377     case KW_TeletexString: return "TeletexString";
 378     case KW_TRUE: return "TRUE";
 379     case KW_UNION: return "UNION";
 380     case KW_UNIQUE: return "UNIQUE";
 381     case KW_UNIVERSAL: return "UNIVERSAL";
 382     case KW_UniversalString: return "UniversalString";
 383     case KW_UTCTime: return "UTCTime";
 384     case KW_UTF8String: return "UTF8String";
 385     case KW_VideotexString: return "VideotexString";
 386     case KW_VisibleString: return "VisibleString";
 387     case KW_WITH: return "WITH";
 388     case TOK_UpperIdentifier: return "<upperidentifier>";
 389     case TOK_LowerIdentifier: return "<loweridentifier>";
 390     case TOK_ampUpperIdentifier: return "<&upperidentifier>";
 391     case TOK_ampLowerIdentifier: return "<&loweridentifier>";
 392     case TOK_Number: return "<number>";
 393     case TOK_RealNumber: return "<realnumber>";
 394     case TOK_BString: return  "<bstring>";
 395     case TOK_CString: return  "<cstring>";
 396     case TOK_HString: return  "<hstring>";
 397     case TOK_Block: return  "<{block}>";
 398     /* special stuff */
 399     case KW_Block_NamedNumberList: return "<NamedNumberList:>";
 400     case KW_Block_Enumerations: return "<Enumerations:>";
 401     case KW_Block_Assignment: return "<Assignment:>";
 402     case KW_Block_NamedBitList: return "<NamedBitList:>";
 403     case KW_Block_IdentifierList: return "<IdentifierList:>";
 404     case KW_Block_FieldSpecList: return "<FieldSpecList:>";
 405     case KW_Block_ComponentTypeLists: return "<ComponentTypeLists:>";
 406     case KW_Block_AlternativeTypeLists: return "<AlternativeTypeLists:>";
 407     case KW_Block_Type: return "<Type:>";
 408     case KW_Block_Value: return "<Value:>";
 409     case KW_Block_ValueSet: return "<ValueSet:>";
 410     case KW_Block_Object: return "<Object:>";
 411     case KW_Block_ObjectSet: return "<ObjectSet:>";
 412     case KW_Block_SeqOfValue: return "<SeqOfValue:>";
 413     case KW_Block_SetOfValue: return "<SetOfValue:>";
 414     case KW_Block_SequenceValue: return "<SequenceValue:>";
 415     case KW_Block_SetValue: return "<SetValue:>";
 416     case KW_Block_ObjectSetSpec: return "<ObjectSetSpec:>";
 417     case KW_Block_DefinedObjectSetBlock: return "<DefinedObjectSetBlock:>";
 418     case KW_Block_AtNotationList: return "<AtNotationList:>";
 419     case KW_Block_OIDValue: return "<OIDValue:>";
 420     case KW_Block_ROIDValue: return "<ROIDValue:>";
 421     case KW_Block_CharStringValue: return "<CharStringValue:>";
 422     case KW_Block_QuadrupleOrTuple: return "<QuadrupleOrTuple:>";
 423     default:
 424       return "<Error! Not a keyword.>";
 425     } // switch token
 426   }
 427
 428   const Identifier& Token::get_semval_id() const
 429   {
 430     switch(token) {
 431     case TOK_UpperIdentifier:
 432     case TOK_LowerIdentifier:
 433     case TOK_ampUpperIdentifier:
 434     case TOK_ampLowerIdentifier:
 435       return *semval.id;
 436     default:
 437       FATAL_ERROR("Token::get_semval_id()");
 438     } // switch
 439   }
 440
 441   void Token::create_block(TokenBuf *tb)
 442   {
 443     if (token != '{' || !tb) FATAL_ERROR("Token::create_block()");
 444     token = TOK_Block;
 445     /** \todo set the correct end line/column info */
 446     semval.block = new Block(tb);
 447     semval.block->set_location(*this);
 448   }
 449
 450   void Token::dump(unsigned level) const
 451   {
 452     switch(token) {
 453     case TOK_UpperIdentifier:
 454     case TOK_LowerIdentifier:
 455     case TOK_ampUpperIdentifier:
 456     case TOK_ampLowerIdentifier:
 457       semval.id->dump(level);
 458       break;
 459     case TOK_Number:
 460       DEBUG(level, "token: number (%s)", semval.i->t_str().c_str());
 461       break;
 462     case TOK_CString:
 463       DEBUG(level, "token: cstring (\"%s\")", semval.str->c_str());
 464       break;
 465     case TOK_RealNumber:
 466     case TOK_BString:
 467     case TOK_HString:
 468       semval.value->dump(level);
 469       break;
 470     case TOK_Block:
 471       semval.block->dump(level);
 472       break;
 473     default:
 474       DEBUG(level, "%s", get_token_name());
 475     }
 476   }
 477
 478   // =================================
 479   // ===== TokenBuf
 480   // =================================
 481
 482   TokenBuf::TokenBuf()
 483     : Node(), filename("<undef>")
 484   {
 485     tokens=new tokens_t();
 486   }
 487
 488   TokenBuf::TokenBuf(tokens_t *p_tokens)
 489     : Node(), tokens(p_tokens), filename("<undef>")
 490   {
 491     if(!p_tokens)
 492       FATAL_ERROR("NULL parameter: Asn::TokenBuf::TokenBuf(tokens_t*)");
 493   }
 494
 495   TokenBuf::TokenBuf(const TokenBuf& p)
 496     : Node(), filename(p.filename)
 497   {
 498     tokens=new tokens_t();
 499     for(size_t i=0; i<p.tokens->size(); i++)
 500       tokens->add((*p.tokens)[i]->clone());
 501   }
 502
 503   TokenBuf::~TokenBuf()
 504   {
 505     delete_tokens();
 506     delete tokens;
 507   }
 508
 509   void TokenBuf::delete_tokens()
 510   {
 511     for(size_t i=0; i<tokens->size(); i++)
 512       delete (*tokens)[i];
 513     tokens->clear();
 514   }
 515
 516   void TokenBuf::reset(const char *p_filename)
 517   {
 518     delete_tokens();
 519     filename=p_filename;
 520   }
 521
 522   bool TokenBuf::read_next()
 523   {
 524     /* if there was an EOF, don't try to read another token */
 525     if(!tokens->empty() && (*tokens)[tokens->size()-1]->get_token()=='\0')
 526       return false;
 527     int token=yylex();
 528     tokens->add(new Token(token, yylval, Location(filename, yylineno)));
 529     return true;
 530   }
 531
 532   Token*& TokenBuf::get_at(size_t pos)
 533   {
 534     while(tokens->size()<=pos && read_next()) ;
 535     if(pos<tokens->size()) {
 536       return (*tokens)[pos];
 537     }
 538     else {
 539       if(tokens->empty())
 540         FATAL_ERROR("Asn::TokenBuf::get_at()");
 541       return (*tokens)[tokens->size()-1];
 542     }
 543   }
 544
 545   void TokenBuf::push_front_token(Token *p_token)
 546   {
 547     if(!p_token)
 548       FATAL_ERROR("Asn::TokenBuf::push_front_token()");
 549     tokens->add_front(p_token);
 550   }
 551
 552   void TokenBuf::push_front_kw_token(int kw)
 553   {
 554     if(tokens->empty())
 555       FATAL_ERROR("Asn::TokenBuf::push_front_kw_token()");
 556     tokens->add_front(new Token(kw, *(*tokens)[0]));
 557   }
 558
 559   void TokenBuf::push_back_token(Token *p_token)
 560   {
 561     if(!p_token)
 562       FATAL_ERROR("NULL parameter: Asn::TokenBuf::push_back_token()");
 563     tokens->add(p_token);
 564   }
 565
 566   void TokenBuf::push_back_kw_token(int kw)
 567   {
 568     if(tokens->empty())
 569       FATAL_ERROR("Asn::TokenBuf::push_back_kw_token()");
 570     tokens->add(new Token(kw, *(*tokens)[tokens->size() - 1]));
 571   }
 572
 573   void TokenBuf::move_tokens_from(TokenBuf *tb)
 574   {
 575     if(!tb)
 576       FATAL_ERROR("NULL parameter: Asn::TokenBuf::move_tokens_from()");
 577     size_t insert_pos=tokens->size();
 578     if(!tokens->empty() && (*tokens)[tokens->size()-1]->get_token()=='\0')
 579       insert_pos--;
 580     size_t num_of_toks=tb->tokens->size();
 581     if(!tb->tokens->empty() &&
 582        (*tb->tokens)[tb->tokens->size()-1]->get_token()=='\0')
 583       num_of_toks--;
 584     tokens_t *tmp_tokens=tb->tokens->subvector(0, num_of_toks);
 585     tb->tokens->replace(0, num_of_toks);
 586     tokens->replace(insert_pos, 0, tmp_tokens);
 587     tmp_tokens->clear(); delete tmp_tokens;
 588   }
 589
 590   Token* TokenBuf::pop_front_token()
 591   {
 592     Token *token=get_at(0);
 593     if(token->get_token()=='{') {
 594       token->set_loc_info(); /* report this line if unmatch_error */
 595       /* search the matching '}' */
 596       int blocklevel=1;
 597       size_t i;
 598       bool unmatch_error=false;
 599       for(i=1; blocklevel>0 && !unmatch_error; i++) {
 600         switch(get_at(i)->get_token()) {
 601         case '{':
 602           blocklevel++;
 603           break;
 604         case '}':
 605           blocklevel--;
 606           break;
 607         case 0:
 608           yyerror("Unmatched '{'");
 609           unmatch_error=true;
 610           i--; /* EOF is not part of the block */
 611           break;
 612         }
 613       }
 614       /* copy the tokens of the block to a new vector */
 615       tokens_t *tokens_block=tokens->subvector(1, i-1);
 616       if(!unmatch_error) /* replace '}' to EOF */
 617         (*tokens_block)[tokens_block->size()-1]->set_token('\0');
 618       else /* clone EOF and add to block */
 619         tokens_block->add(get_at(i)->clone());
 620       /* remove the tokens from their original vector */
 621       tokens->replace(0, i);
 622       /* create the new tokenbuf from the tokens of the block */
 623       TokenBuf *tb=new TokenBuf(tokens_block);
 624       tb->filename=filename;
 625       /* transform the opening '{' to a block-token */
 626       token->create_block(tb);
 627     }
 628     else {
 629       /* remove the first token */
 630       tokens->replace(0, 1);
 631     }
 632     return token;
 633   }
 634
 635   int TokenBuf::lex()
 636   {
 637     Token *token=pop_front_token();
 638     int tmp_tok = token->get_token();
 639     token->set_loc_info();
 640     token->steal_semval(yylval);
 641     yylloc.first_line=yylloc.last_line=token->get_lineno();
 642     yylloc.first_column=yylloc.last_column=0;
 643     delete token;
 644     return tmp_tok;
 645   }
 646
 647   void TokenBuf::dump(unsigned level) const
 648   {
 649     DEBUG(level, "Tokens: (%lu pcs.)", (unsigned long) tokens->size());
 650     level++;
 651     for(size_t i=0; i<tokens->size(); i++)
 652       (*tokens)[i]->dump(level);
 653   }
 654
 655 } // namespace Asn