From e38245261e19dd394edc02f9db366d5770cc5f9b Mon Sep 17 00:00:00 2001 From: rexy712 Date: Sat, 14 Mar 2020 09:21:57 -0700 Subject: [PATCH] Remove parsing dependance on null terminator --- src/input.c | 152 +++++++++++++++++++++++----------------------------- 1 file changed, 67 insertions(+), 85 deletions(-) diff --git a/src/input.c b/src/input.c index b69dbd5..a1a89fe 100644 --- a/src/input.c +++ b/src/input.c @@ -30,7 +30,7 @@ #define RJP_INITIAL_PARSE_DEPTH 16 #define rjp_lex_accept 1 -typedef enum RJP_lex_node{ +typedef enum RJP_lex_category{ rjp_lex_start = 0, rjp_lex_obracket = 3, rjp_lex_obrace = 5, @@ -65,31 +65,12 @@ typedef enum RJP_lex_node{ rjp_lex_signed_number = 49, rjp_lex_sci_num_signed = 51, rjp_lex_newlines = 53, - rjp_lex_block_comment = 55, + rjp_lex_block_comment_start = 54, rjp_lex_block_comment_end1 = 56, - rjp_lex_block_comment_end2 = 57, + rjp_lex_block_comment = 57, rjp_lex_invalid = 1000, - rjp_lex_unrecognized_word = 1002 -}RJP_lex_node; - -typedef enum RJP_lex_category{ - rjp_lexcat_invalid, - rjp_lexcat_obrace, - rjp_lexcat_cbrace, - rjp_lexcat_obracket, - rjp_lexcat_cbracket, - rjp_lexcat_space, - rjp_lexcat_newline, - rjp_lexcat_string, - rjp_lexcat_colon, - rjp_lexcat_comma, - rjp_lexcat_integer, - rjp_lexcat_float, - rjp_lexcat_scientific, - rjp_lexcat_true, - rjp_lexcat_false, - rjp_lexcat_null, - rjp_lexcat_comment, + rjp_lex_unrecognized_word = 1002, + rjp_lex_end = 1004, }RJP_lex_category; typedef enum RJP_yacc_target{ @@ -107,7 +88,7 @@ typedef enum RJP_yacc_target{ typedef struct RJP_lex_state{ const char* str; - RJP_lex_node node; + RJP_lex_category node; RJP_index length; RJP_index offset; }RJP_lex_state; @@ -164,7 +145,7 @@ static RJP_lex_category irjp_lex_accept(RJP_lex_category val, RJP_lex_state* sta return val; } static RJP_lex_category irjp_lex_error(RJP_lex_state* state){ - return irjp_lex_accept(rjp_lexcat_invalid, state); + return irjp_lex_accept(rjp_lex_invalid, state); } static RJP_lex_category irjp_lex(RJP_lex_state* state){ state->offset += state->length; @@ -203,31 +184,33 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ state->node = rjp_lex_n; else if(ch == '/') state->node = rjp_lex_slash; + else if(ch == 0) + return irjp_lex_accept(rjp_lex_end, state); else state->node = rjp_lex_invalid; break; //punctuation case rjp_lex_obracket: - return irjp_lex_accept(rjp_lexcat_obracket, state); + return irjp_lex_accept(rjp_lex_obracket, state); case rjp_lex_obrace: - return irjp_lex_accept(rjp_lexcat_obrace, state); + return irjp_lex_accept(rjp_lex_obrace, state); case rjp_lex_cbracket: - return irjp_lex_accept(rjp_lexcat_cbracket, state); + return irjp_lex_accept(rjp_lex_cbracket, state); case rjp_lex_cbrace: - return irjp_lex_accept(rjp_lexcat_cbrace, state); + return irjp_lex_accept(rjp_lex_cbrace, state); case rjp_lex_comma: - return irjp_lex_accept(rjp_lexcat_comma, state); + return irjp_lex_accept(rjp_lex_comma, state); case rjp_lex_colon: - return irjp_lex_accept(rjp_lexcat_colon, state); + return irjp_lex_accept(rjp_lex_colon, state); //whitespace case rjp_lex_newlines: - return irjp_lex_accept(rjp_lexcat_newline, state); + return irjp_lex_accept(rjp_lex_newlines, state); case rjp_lex_spaces: if(isspace(ch)) break; - return irjp_lex_accept(rjp_lexcat_space, state); + return irjp_lex_accept(rjp_lex_spaces, state); //numbers case rjp_lex_signed_number: @@ -244,7 +227,7 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ else if(isalpha(ch)) state->node = rjp_lex_unrecognized_word; else - return irjp_lex_accept(rjp_lexcat_integer, state); + return irjp_lex_accept(rjp_lex_number, state); } break; case rjp_lex_decimal: @@ -261,7 +244,7 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ else if(isalpha(ch)) state->node = rjp_lex_unrecognized_word; else - return irjp_lex_accept(rjp_lexcat_float, state); + return irjp_lex_accept(rjp_lex_fnumber, state); break; case rjp_lex_fnum_e: if(ch == '-' || ch == '+') @@ -283,7 +266,7 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ else if(isalpha(ch)) state->node = rjp_lex_unrecognized_word; else - return irjp_lex_accept(rjp_lexcat_scientific, state); + return irjp_lex_accept(rjp_lex_sci_num, state); break; //strings @@ -299,33 +282,33 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ state->node = rjp_lex_quote; break; case rjp_lex_string: - return irjp_lex_accept(rjp_lexcat_string, state); + return irjp_lex_accept(rjp_lex_string, state); //comments case rjp_lex_slash: if(ch == '/') state->node = rjp_lex_line_comment; else if(ch == '*') - state->node = rjp_lex_block_comment; + state->node = rjp_lex_block_comment_start; else state->node = rjp_lex_invalid; break; case rjp_lex_line_comment: if(ch == '\n' || ch == '\r' || ch == 0) //don't consume this character - return irjp_lex_accept(rjp_lexcat_comment, state); + return irjp_lex_accept(rjp_lex_line_comment, state); break; - case rjp_lex_block_comment: + case rjp_lex_block_comment_start: if(ch == '*') state->node = rjp_lex_block_comment_end1; break; case rjp_lex_block_comment_end1: if(ch == '/') - state->node = rjp_lex_block_comment_end2; - else state->node = rjp_lex_block_comment; + else + state->node = rjp_lex_block_comment_start; break; - case rjp_lex_block_comment_end2: - return irjp_lex_accept(rjp_lexcat_comment, state); + case rjp_lex_block_comment: + return irjp_lex_accept(rjp_lex_block_comment, state); //true case rjp_lex_t: @@ -348,7 +331,7 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ break; case rjp_lex_true: if(!isalnum(ch)) - return irjp_lex_accept(rjp_lexcat_true, state); + return irjp_lex_accept(rjp_lex_true, state); else state->node = rjp_lex_unrecognized_word; break; @@ -380,7 +363,7 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ break; case rjp_lex_false: if(!isalnum(ch)) - return irjp_lex_accept(rjp_lexcat_false, state); + return irjp_lex_accept(rjp_lex_false, state); state->node = rjp_lex_unrecognized_word; break; @@ -405,7 +388,7 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ break; case rjp_lex_null: if(!isalnum(ch)) - return irjp_lex_accept(rjp_lexcat_null, state); + return irjp_lex_accept(rjp_lex_null, state); state->node = rjp_lex_unrecognized_word; break; @@ -421,7 +404,7 @@ static RJP_lex_category irjp_lex(RJP_lex_state* state){ if(ch == 0) break; } - return irjp_lex_error(state); + return irjp_lex_accept(state->node, state); } static int irjp_init_value(RJP_value* newval, RJP_lex_category cat, RJP_yacc_state* state){ @@ -429,7 +412,7 @@ static int irjp_init_value(RJP_value* newval, RJP_lex_category cat, RJP_yacc_sta RJP_index offset = state->lexstate.offset; const char* str = state->lexstate.str + offset; switch(cat){ - case rjp_lexcat_string:; + case rjp_lex_string:; RJP_index newlength; newval->type = rjp_json_string; newval->string.value = irjp_convert_string(str, length, &newlength); @@ -437,32 +420,32 @@ static int irjp_init_value(RJP_value* newval, RJP_lex_category cat, RJP_yacc_sta return 1; newval->string.length = newlength; break; - case rjp_lexcat_integer: + case rjp_lex_number: newval->type = rjp_json_integer; newval->integer = strtoll(str, NULL, 10); break; - case rjp_lexcat_float: - case rjp_lexcat_scientific: + case rjp_lex_fnumber: + case rjp_lex_sci_num: newval->type = rjp_json_dfloat; newval->dfloat = strtod(str, NULL); break; - case rjp_lexcat_true: + case rjp_lex_true: newval->type = rjp_json_boolean; newval->boolean = 1; break; - case rjp_lexcat_false: + case rjp_lex_false: newval->type = rjp_json_boolean; newval->boolean = 0; break; - case rjp_lexcat_null: + case rjp_lex_null: newval->type = rjp_json_null; break; - case rjp_lexcat_obrace: + case rjp_lex_obrace: newval->type = rjp_json_object; irjp_yacc_stack_push(&state->target_stack, rjp_yacc_first_mem_key); state->curr = state->lastadded; break; - case rjp_lexcat_obracket: + case rjp_lex_obracket: newval->type = rjp_json_array; irjp_yacc_stack_push(&state->target_stack, rjp_yacc_arr_first_value); state->curr = state->lastadded; @@ -488,8 +471,8 @@ static RJP_value* irjp_add_value_to_object(RJP_yacc_state* state, const char* ke static RJP_lex_category irjp_convert_comment(_Bool allow_comments){ if(allow_comments) - return rjp_lexcat_space; - return rjp_lexcat_invalid; + return rjp_lex_spaces; + return rjp_lex_invalid; } static void irjp_init_parse_state(RJP_yacc_state* state, const char* str){ @@ -514,31 +497,18 @@ static void irjp_delete_parse_state(RJP_yacc_state* state){ return RJP_PARSE_STATUS_ERR; \ }while(0) -static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state); -static int irjp_parse(RJP_yacc_state* state){ - RJP_lex_category cat; - for(;state->lexstate.str[state->lexstate.offset+state->lexstate.length];state->row += state->lexstate.length){ - cat = irjp_lex(&state->lexstate); - if(irjp_parse_handle_lexcat(cat, state) != RJP_PARSE_STATUS_SUC) - return RJP_PARSE_STATUS_ERR; - } - if(state->target_stack.position != 0) - irjp_parse_error("Missing closing brace"); - return RJP_PARSE_STATUS_SUC; -} - static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state){ - if(cat == rjp_lexcat_comment) + if(cat == rjp_lex_line_comment || cat == rjp_lex_block_comment) cat = irjp_convert_comment(state->allow_comments); - if(cat == rjp_lexcat_space) + if(cat == rjp_lex_spaces) return RJP_PARSE_STATUS_SUC; - if(cat == rjp_lexcat_newline){ + if(cat == rjp_lex_newlines){ state->row = 1; ++(state->column); return RJP_PARSE_STATUS_SUC; } - if(cat == rjp_lexcat_invalid) + if(cat == rjp_lex_invalid) irjp_parse_error("Invalid token"); switch(irjp_yacc_stack_current(&state->target_stack)){ @@ -550,13 +520,13 @@ static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state) } break; case rjp_yacc_first_mem_key: - if(cat == rjp_lexcat_cbrace){ + if(cat == rjp_lex_cbrace){ irjp_yacc_stack_pop(&state->target_stack); state->curr = state->curr->parent; }else{ //fallthrough case rjp_yacc_mem_key: - if(cat == rjp_lexcat_string){ + if(cat == rjp_lex_string){ irjp_yacc_stack_set(&state->target_stack, rjp_yacc_key_colon); if(!irjp_add_value_to_object(state, state->lexstate.str+state->lexstate.offset, state->lexstate.length)){ irjp_parse_error("Expected member key"); @@ -567,7 +537,7 @@ static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state) } break; case rjp_yacc_arr_first_value: - if(cat == rjp_lexcat_cbracket){ + if(cat == rjp_lex_cbracket){ irjp_yacc_stack_pop(&state->target_stack); state->curr = state->curr->parent; }else{ @@ -580,7 +550,7 @@ static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state) break; case rjp_yacc_key_colon: - if(cat != rjp_lexcat_colon) + if(cat != rjp_lex_colon) irjp_parse_error("Expected member key"); irjp_yacc_stack_set(&state->target_stack, rjp_yacc_obj_value); break; @@ -591,9 +561,9 @@ static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state) } break; case rjp_yacc_obj_comma: - if(cat == rjp_lexcat_comma){ + if(cat == rjp_lex_comma){ irjp_yacc_stack_set(&state->target_stack, state->allow_trail_comma ? rjp_yacc_first_mem_key : rjp_yacc_mem_key); - }else if(cat == rjp_lexcat_cbrace){ + }else if(cat == rjp_lex_cbrace){ irjp_yacc_stack_pop(&state->target_stack); state->curr = state->curr->parent; }else{ @@ -602,9 +572,9 @@ static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state) break; case rjp_yacc_arr_comma: - if(cat == rjp_lexcat_comma){ + if(cat == rjp_lex_comma){ irjp_yacc_stack_set(&state->target_stack, state->allow_trail_comma ? rjp_yacc_arr_first_value : rjp_yacc_arr_value); - }else if(cat == rjp_lexcat_cbracket){ + }else if(cat == rjp_lex_cbracket){ irjp_yacc_stack_pop(&state->target_stack); state->curr = state->curr->parent; }else{ @@ -618,6 +588,18 @@ static int irjp_parse_handle_lexcat(RJP_lex_category cat, RJP_yacc_state* state) }; return RJP_PARSE_STATUS_SUC; } +static int irjp_parse(RJP_yacc_state* state){ + RJP_lex_category cat; + for(cat = irjp_lex(&state->lexstate);cat & rjp_lex_accept;cat = irjp_lex(&state->lexstate),state->row += state->lexstate.length){ + if(irjp_parse_handle_lexcat(cat, state) != RJP_PARSE_STATUS_SUC) + return RJP_PARSE_STATUS_ERR; + } + if(state->target_stack.position != 0) + irjp_parse_error("Missing closing brace"); + if(cat == rjp_lex_end) + return RJP_PARSE_STATUS_SUC; + irjp_parse_error("Invalid Token"); +} RJP_value* rjp_parse(const char* str, int flags){ RJP_yacc_state state = {.allow_comments = (flags & RJP_PARSE_ALLOW_COMMENTS), .allow_trail_comma = (flags & RJP_PARSE_ALLOW_TRAILING_COMMA)