rexy712/rjp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

Add callback based reading, essentially making chunked reading redundant

Browse Source
This commit is contained in:
rexy712
2020-03-22 11:14:10 -07:00
parent 841c179c24
commit f88bb0c4f2
6 changed files with 250 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
TODO
View File

@@ -1 +0,0 @@
Implement chunked reading

6
include/rjp.h
View File

@@ -89,6 +89,10 @@ typedef enum RJP_parse_flag{
RJP_PARSE_ALLOW_COMMENTS = 1,
RJP_PARSE_ALLOW_TRAILING_COMMA = 2
}RJP_parse_flag;
typedef struct RJP_parse_callback{
int(*read)(char*,int,void*);
void* data;
}RJP_parse_callback;
/***************** NON OBJECT OPERATIONS *******************/
@@ -109,7 +113,7 @@ RJP_index rjp_escape_strlen(const char* str);
/***************** GENERIC OPERATIONS *******************/
//Convert C string consisting of json data into RJP's format
RJP_value* rjp_parse(const char* str, int flags);
//RJP_value* rjp_parse_chunked(const char* str, RJP_value* prev_chunk);
RJP_value* rjp_parse_cback(int flags, RJP_parse_callback* cbacks);
char* rjp_to_json(const RJP_value* root, int pretty);
//Initialize a root RJP_value to copy of value

12
include/rjp_lex.h
View File

@@ -21,6 +21,9 @@
#include "rjp.h"
#define RJP_LEX_CBACK_BUFFER_SIZE 512
#define RJP_LEX_CBACK_STR_SIZE 64
#define rjp_lex_accept 1
typedef enum RJP_lex_category{
rjp_lex_start = 0,
@@ -66,12 +69,19 @@ typedef enum RJP_lex_category{
}RJP_lex_category;
typedef struct RJP_lex_state{
const char* str;
char* str;
char* buff;
RJP_index strl;
RJP_index buffl;
RJP_index buffcap;
RJP_index buffpos;
RJP_lex_category node;
RJP_index length;
RJP_index offset;
}RJP_lex_state;
void irjp_init_lex_cback_state(RJP_lex_state* state);
RJP_lex_category irjp_lex(RJP_lex_state* state);
RJP_lex_category irjp_lex_cback(RJP_lex_state* state, RJP_parse_callback* cbacks);
#endif

48
src/rjp_lex.c
View File

@@ -19,7 +19,15 @@
#include "rjp_lex.h"
#include "rjp.h"
#include <ctype.h> //isalpha, etc
#include <string.h> //memcpy
void irjp_init_lex_cback_state(RJP_lex_state* state){
state->str = rjp_alloc(RJP_LEX_CBACK_STR_SIZE+1);
state->str[RJP_LEX_CBACK_STR_SIZE] = 0;
state->strl = RJP_LEX_CBACK_STR_SIZE;
state->buff = rjp_alloc(RJP_LEX_CBACK_BUFFER_SIZE);
state->buffcap = RJP_LEX_CBACK_BUFFER_SIZE;
}
static RJP_lex_category irjp_lex_accept(RJP_lex_category val, RJP_lex_state* state){
state->node = rjp_lex_start;
return val;
@@ -297,3 +305,43 @@ RJP_lex_category irjp_lex(RJP_lex_state* state){
}
return irjp_lex_accept(state->node, state);
}
static void irjp_lex_resize_strbuf(RJP_lex_state* state, int newsize){
char* newbuf = rjp_alloc(newsize+1);
memcpy(newbuf, state->str, state->strl);
newbuf[newsize] = 0;
rjp_free(state->str);
state->str = newbuf;
state->strl = newsize;
}
RJP_lex_category irjp_lex_cback(RJP_lex_state* state, RJP_parse_callback* cbacks){
state->length = 0;
RJP_index chars_read = state->buffl;
if(chars_read == 0){
state->buffpos = 0;
chars_read = cbacks->read(state->buff, state->buffcap, cbacks->data);
state->buffl = chars_read;
}
while(chars_read > 0){
for(RJP_index i = 0;(i+state->buffpos) < chars_read;++i,++state->length){
if(state->length == state->strl)
irjp_lex_resize_strbuf(state, state->strl*2);
RJP_lex_category cat = irjp_lex_char(state->buff[state->buffpos+i], state->node);
if(cat == rjp_lex_invalid){
state->buffpos = i + state->buffpos;
state->str[state->length] = 0;
return irjp_lex_accept(state->node, state);
}
state->str[state->length] = state->buff[state->buffpos+i];
state->node = cat;
}
chars_read = cbacks->read(state->buff, state->buffcap, cbacks->data);
state->buffpos = 0;
state->buffl = chars_read;
}
++state->buffpos;
state->str[state->length] = 0;
RJP_lex_category cat = state->node;
state->node = rjp_lex_end;
return cat;
}

33
src/rjp_parse.c
View File

@@ -164,7 +164,7 @@ static void irjp_init_parse_state(RJP_parse_state* state, const char* str){
state->row = 1;
irjp_init_parse_stack(&state->target_stack);
state->lexstate.str = str;
state->lexstate.str = (char*)str;
state->root = state->curr = state->lastadded = rjp_calloc(1, sizeof(RJP_value));
}
static void irjp_delete_parse_state(RJP_parse_state* state){
@@ -284,6 +284,18 @@ static int irjp_parse(RJP_parse_state* state){
return RJP_PARSE_STATUS_SUC;
irjp_parse_error("Invalid Token");
}
static int irjp_parse_cback(RJP_parse_state* state, RJP_parse_callback* cback){
RJP_lex_category cat;
for(cat = irjp_lex_cback(&state->lexstate, cback);cat & rjp_lex_accept;cat = irjp_lex_cback(&state->lexstate, cback),state->row += state->lexstate.length){
if(irjp_parse_handle_lexcat(cat, state) != RJP_PARSE_STATUS_SUC)
return RJP_PARSE_STATUS_ERR;
}
if(state->target_stack.position != 0)
irjp_parse_error("Missing closing brace");
if(cat == rjp_lex_end)
return RJP_PARSE_STATUS_SUC;
irjp_parse_error("Invalid Token");
}
RJP_value* rjp_parse(const char* str, int flags){
RJP_parse_state state = {.allow_comments = (flags & RJP_PARSE_ALLOW_COMMENTS),
.allow_trail_comma = (flags & RJP_PARSE_ALLOW_TRAILING_COMMA)
@@ -299,9 +311,20 @@ RJP_value* rjp_parse(const char* str, int flags){
}
}
RJP_value* rjp_parse_chunked(const char* str, RJP_value* prev_chunk){
if(!prev_chunk){
return rjp_parse(str, RJP_PARSE_NONE);
RJP_value* rjp_parse_cback(int flags, RJP_parse_callback* cback){
RJP_parse_state state = {.allow_comments = (flags & RJP_PARSE_ALLOW_COMMENTS),
.allow_trail_comma = (flags & RJP_PARSE_ALLOW_TRAILING_COMMA)
};
irjp_init_parse_state(&state, NULL);
irjp_init_lex_cback_state(&state.lexstate);
int status = irjp_parse_cback(&state, cback);
rjp_free(state.lexstate.str);
rjp_free(state.lexstate.buff);
if(status == RJP_PARSE_STATUS_SUC){
irjp_delete_parse_stack(&state.target_stack);
return state.root;
}else{
irjp_delete_parse_state(&state);
return NULL;
}
return NULL;
}

269
src/test/test.c
View File

@@ -3,10 +3,29 @@
#include "rjp_string.h"
#include <stdio.h>
#include <string.h>
int test(const char* str, RJP_parse_flag flags){
RJP_value* res;
res = rjp_parse(str, flags);
#define MIN(x, y) ((x < y) ? (x) : (y))
typedef struct parse_stuff{
const char* str;
int len;
int pos;
}parse_stuff;
int read_callback(char* c, int size, void* userdata){
parse_stuff* data = (parse_stuff*)userdata;
if(data->pos >= data->len)
return 0;
int min = MIN(size, data->len - data->pos);
int i;
for(i = 0;i < min;++i){
c[i] = data->str[data->pos++];
}
return i;
}
int handle_res(RJP_value* res){
if(res){
fprintf(stderr, "Accepted\n");
}
@@ -19,123 +38,141 @@ int test(const char* str, RJP_parse_flag flags){
rjp_free_value(res);
return failed;
}
int test_cbacks(const char* str, RJP_parse_flag flags){
RJP_parse_callback cbacks;
parse_stuff cback_data = {str, strlen(str), 0};
cbacks.read = read_callback;
cbacks.data = &cback_data;
RJP_value* res;
res = rjp_parse_cback(flags, &cbacks);
return handle_res(res);
}
int test(const char* str, RJP_parse_flag flags){
RJP_value* res;
res = rjp_parse(str, flags);
return handle_res(res);
}
struct parse_pair{
const char* str;
RJP_parse_flag flags;
};
int main(){
struct parse_pair should_pass_strings[] = {
{"{}", RJP_PARSE_NONE},
{"[]", RJP_PARSE_NONE},
{"\"s\"", RJP_PARSE_NONE},
{"\"\\n\"", RJP_PARSE_NONE},
{"\"\\\"\"", RJP_PARSE_NONE},
{"\"str\\nstr\"", RJP_PARSE_NONE},
{"\"\\uD83D\\uDE10\"", RJP_PARSE_NONE},
{"true", RJP_PARSE_NONE},
{"false", RJP_PARSE_NONE},
{"null", RJP_PARSE_NONE},
{"5", RJP_PARSE_NONE},
{"-5", RJP_PARSE_NONE},
{"+5", RJP_PARSE_NONE},
{"5.5", RJP_PARSE_NONE},
{"-5.5", RJP_PARSE_NONE},
{"+5.5", RJP_PARSE_NONE},
{"5.5e6", RJP_PARSE_NONE},
{"-5.5e6", RJP_PARSE_NONE},
{"+5.5e6", RJP_PARSE_NONE},
{"5.5e+6", RJP_PARSE_NONE},
{"-5.5e+6", RJP_PARSE_NONE},
{"+5.5e+6", RJP_PARSE_NONE},
{"5.5e-6", RJP_PARSE_NONE},
{"-5.5e-6", RJP_PARSE_NONE},
{"+5.5e-6", RJP_PARSE_NONE},
{" {}", RJP_PARSE_NONE},
{"\n{}\n", RJP_PARSE_NONE},
{" { \"key\" \t:\n\n\n5 \n\t\n } ", RJP_PARSE_NONE},
{" {\t }\n", RJP_PARSE_NONE},
{"5.5 ", RJP_PARSE_NONE},
{"{\"key\":5}", RJP_PARSE_NONE},
{"{\"key\":{}}", RJP_PARSE_NONE},
{"{\"\\uD83D\\uDE10\":5}", RJP_PARSE_NONE},
{"{\"😐\":5}", RJP_PARSE_NONE},
{"{\"key\":{\"key\":5}}", RJP_PARSE_NONE},
{"{\"key\":{\"key\":5,\"key2\":6}}", RJP_PARSE_NONE},
{"{\"key\":{\"key\":5},\"key2\":6}", RJP_PARSE_NONE},
{"[5, 6, 7, 8, 9, \"10\"]", RJP_PARSE_NONE},
{"[[5,6],[7,8],[9,\"10\"]]", RJP_PARSE_NONE},
{"{\"arr\":[5,6,6]}", RJP_PARSE_NONE},
{"[{\"arr\":[5,6,6]}]", RJP_PARSE_NONE},
{"[{\"arr\":[5,6,6]}, 6]", RJP_PARSE_NONE},
{"[5,6,6,6,6.6]", RJP_PARSE_NONE},
{"[6,7,]", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"{\"1\":1,\"2\":2,}", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"[6,]", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"{\"1\":1,}", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"//comment\n{}", RJP_PARSE_ALLOW_COMMENTS},
{"{\"key\"://comment\n5}", RJP_PARSE_ALLOW_COMMENTS},
{"{\"key\"//comment\n:5}", RJP_PARSE_ALLOW_COMMENTS},
{"{}//comment", RJP_PARSE_ALLOW_COMMENTS},
{"{//\"key\":5\n}", RJP_PARSE_ALLOW_COMMENTS},
{"5 //comment*/", RJP_PARSE_ALLOW_COMMENTS},
{"{/*\"key\":5*/\"key\":5}", RJP_PARSE_ALLOW_COMMENTS},
{"[5, /*comment*/6]", RJP_PARSE_ALLOW_COMMENTS},
};
const int should_pass_cnt = sizeof(should_pass_strings)/sizeof(should_pass_strings[0]);
struct parse_pair should_fail_strings[] = {
{"//comment\n{}", RJP_PARSE_NONE},
{"{", RJP_PARSE_NONE},
{"}", RJP_PARSE_NONE},
{"[", RJP_PARSE_NONE},
{"]", RJP_PARSE_NONE},
{"6.", RJP_PARSE_NONE},
{"6.6e", RJP_PARSE_NONE},
{"6.6e+", RJP_PARSE_NONE},
{"{6}", RJP_PARSE_NONE},
{"[\"key\":5]", RJP_PARSE_NONE},
{"\"string\n\"", RJP_PARSE_NONE},
{"[3 4]", RJP_PARSE_NONE},
{"\"\\uD83D\\uDE1\"", RJP_PARSE_NONE},
{"\"\\uD83D\\uDE1Q\"", RJP_PARSE_NONE},
{"\"\\uD83\\uDE10\"", RJP_PARSE_NONE},
{"\"\\uF83D\\uDE10\"", RJP_PARSE_NONE},
{"\"\\uU83D\\uDE10\"", RJP_PARSE_NONE},
{"{\"key\":1 \"key2\":2}", RJP_PARSE_NONE},
{"{\"key\" 1}", RJP_PARSE_NONE},
{"6, 7", RJP_PARSE_NONE},
{"[,]", RJP_PARSE_NONE},
{"{, RJP_PARSE_NONE}", RJP_PARSE_NONE},
{"[1, 2],", RJP_PARSE_NONE},
{"{\"key\nkey\":5}", RJP_PARSE_NONE},
{"{\"key\":\"key\n\"}", RJP_PARSE_NONE},
{"[6,7,]", RJP_PARSE_NONE},
{"{\"1\":1,\"2\":2, RJP_PARSE_NONE}", RJP_PARSE_NONE},
{"[6,]", RJP_PARSE_NONE},
{"{\"1\":1, RJP_PARSE_NONE}", RJP_PARSE_NONE},
{"{//comment\"key\":\n5}", RJP_PARSE_NONE},
{"{/*\"key\":*/5}", RJP_PARSE_NONE},
{"[5, /*6*/, 7]", RJP_PARSE_NONE},
{"{/*comment}", RJP_PARSE_NONE},
{"{//comment}", RJP_PARSE_NONE},
{"{\"key\"://comment\n5}", RJP_PARSE_NONE},
{"{\"key\"//comment\n:5}", RJP_PARSE_NONE},
{"{}//comment", RJP_PARSE_NONE},
{"{//\"key\":5\n}", RJP_PARSE_NONE},
{"5 //comment*/", RJP_PARSE_NONE},
{"{/*\"key\":5*/\"key\":5}", RJP_PARSE_NONE},
{"[5, /*comment*/6]", RJP_PARSE_NONE},
};
const int should_fail_cnt = sizeof(should_fail_strings)/sizeof(should_fail_strings[0]);
const int total_tests = should_pass_cnt + should_fail_cnt;
struct parse_pair should_pass_strings[] = {
{"{}", RJP_PARSE_NONE},
{"[]", RJP_PARSE_NONE},
{"\"s\"", RJP_PARSE_NONE},
{"\"\\n\"", RJP_PARSE_NONE},
{"\"\\\"\"", RJP_PARSE_NONE},
{"\"str\\nstr\"", RJP_PARSE_NONE},
{"\"\\uD83D\\uDE10\"", RJP_PARSE_NONE},
{"true", RJP_PARSE_NONE},
{"false", RJP_PARSE_NONE},
{"null", RJP_PARSE_NONE},
{"5", RJP_PARSE_NONE},
{"-5", RJP_PARSE_NONE},
{"+5", RJP_PARSE_NONE},
{"5.5", RJP_PARSE_NONE},
{"-5.5", RJP_PARSE_NONE},
{"+5.5", RJP_PARSE_NONE},
{"5.5e6", RJP_PARSE_NONE},
{"-5.5e6", RJP_PARSE_NONE},
{"+5.5e6", RJP_PARSE_NONE},
{"5.5e+6", RJP_PARSE_NONE},
{"-5.5e+6", RJP_PARSE_NONE},
{"+5.5e+6", RJP_PARSE_NONE},
{"5.5e-6", RJP_PARSE_NONE},
{"-5.5e-6", RJP_PARSE_NONE},
{"+5.5e-6", RJP_PARSE_NONE},
{" {}", RJP_PARSE_NONE},
{"\n{}\n", RJP_PARSE_NONE},
{" { \"key\" \t:\n\n\n5 \n\t\n } ", RJP_PARSE_NONE},
{" {\t }\n", RJP_PARSE_NONE},
{"5.5 ", RJP_PARSE_NONE},
{"{\"key\":5}", RJP_PARSE_NONE},
{"{\"key\":{}}", RJP_PARSE_NONE},
{"{\"\\uD83D\\uDE10\":5}", RJP_PARSE_NONE},
{"{\"😐\":5}", RJP_PARSE_NONE},
{"{\"key\":{\"key\":5}}", RJP_PARSE_NONE},
{"{\"key\":{\"key\":5,\"key2\":6}}", RJP_PARSE_NONE},
{"{\"key\":{\"key\":5},\"key2\":6}", RJP_PARSE_NONE},
{"[5, 6, 7, 8, 9, \"10\"]", RJP_PARSE_NONE},
{"[[5,6],[7,8],[9,\"10\"]]", RJP_PARSE_NONE},
{"{\"arr\":[5,6,6]}", RJP_PARSE_NONE},
{"[{\"arr\":[5,6,6]}]", RJP_PARSE_NONE},
{"[{\"arr\":[5,6,6]}, 6]", RJP_PARSE_NONE},
{"[5,6,6,6,6.6]", RJP_PARSE_NONE},
{"[6,7,]", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"{\"1\":1,\"2\":2,}", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"[6,]", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"{\"1\":1,}", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"//comment\n{}", RJP_PARSE_ALLOW_COMMENTS},
{"{\"key\"://comment\n5}", RJP_PARSE_ALLOW_COMMENTS},
{"{\"key\"//comment\n:5}", RJP_PARSE_ALLOW_COMMENTS},
{"{}//comment", RJP_PARSE_ALLOW_COMMENTS},
{"{//\"key\":5\n}", RJP_PARSE_ALLOW_COMMENTS},
{"5 //comment*/", RJP_PARSE_ALLOW_COMMENTS},
{"{/*\"key\":5*/\"key\":5}", RJP_PARSE_ALLOW_COMMENTS},
{"[5, /*comment*/6]", RJP_PARSE_ALLOW_COMMENTS},
};
const int should_pass_cnt = sizeof(should_pass_strings)/sizeof(should_pass_strings[0]);
struct parse_pair should_fail_strings[] = {
{"//comment\n{}", RJP_PARSE_NONE},
{"{", RJP_PARSE_NONE},
{"}", RJP_PARSE_NONE},
{"[", RJP_PARSE_NONE},
{"]", RJP_PARSE_NONE},
{"6.", RJP_PARSE_NONE},
{"6.6e", RJP_PARSE_NONE},
{"6.6e+", RJP_PARSE_NONE},
{"5e", RJP_PARSE_NONE},
{"{6}", RJP_PARSE_NONE},
{"[\"key\":5]", RJP_PARSE_NONE},
{"\"string\n\"", RJP_PARSE_NONE},
{"[3 4]", RJP_PARSE_NONE},
{"\"\\uD83D\\uDE1\"", RJP_PARSE_NONE},
{"\"\\uD83D\\uDE1Q\"", RJP_PARSE_NONE},
{"\"\\uD83\\uDE10\"", RJP_PARSE_NONE},
{"\"\\uF83D\\uDE10\"", RJP_PARSE_NONE},
{"\"\\uU83D\\uDE10\"", RJP_PARSE_NONE},
{"{\"key\":1 \"key2\":2}", RJP_PARSE_NONE},
{"{\"key\" 1}", RJP_PARSE_NONE},
{"6, 7", RJP_PARSE_NONE},
{"[,]", RJP_PARSE_NONE},
{"{, RJP_PARSE_NONE}", RJP_PARSE_NONE},
{"[1, 2],", RJP_PARSE_NONE},
{"{\"key\nkey\":5}", RJP_PARSE_NONE},
{"{\"key\":\"key\n\"}", RJP_PARSE_NONE},
{"[6,7,]", RJP_PARSE_NONE},
{"{\"1\":1,\"2\":2, RJP_PARSE_NONE}", RJP_PARSE_NONE},
{"[6,]", RJP_PARSE_NONE},
{"{\"1\":1, RJP_PARSE_NONE}", RJP_PARSE_NONE},
{"{//comment\"key\":\n5}", RJP_PARSE_NONE},
{"{/*\"key\":*/5}", RJP_PARSE_NONE},
{"[5, /*6*/, 7]", RJP_PARSE_NONE},
{"{/*comment}", RJP_PARSE_NONE},
{"{//comment}", RJP_PARSE_NONE},
{"{\"key\"://comment\n5}", RJP_PARSE_NONE},
{"{\"key\"//comment\n:5}", RJP_PARSE_NONE},
{"{}//comment", RJP_PARSE_NONE},
{"{//\"key\":5\n}", RJP_PARSE_NONE},
{"5 //comment*/", RJP_PARSE_NONE},
{"{/*\"key\":5*/\"key\":5}", RJP_PARSE_NONE},
{"[5, /*comment*/6]", RJP_PARSE_NONE},
{"{\"key\"//:5}", RJP_PARSE_ALLOW_COMMENTS},
{"{,}", RJP_PARSE_ALLOW_TRAILING_COMMA},
{"[,]", RJP_PARSE_ALLOW_TRAILING_COMMA},
};
const int should_fail_cnt = sizeof(should_fail_strings)/sizeof(should_fail_strings[0]);
const int total_tests = should_pass_cnt + should_fail_cnt;
int run_test(int (*fun)(const char*,RJP_parse_flag)){
int passed = 0;
fprintf(stderr, "Running %d tests that should pass...\n", should_pass_cnt);
for(unsigned i = 0;i < sizeof(should_pass_strings)/sizeof(should_pass_strings[0]);++i){
fprintf(stderr, "%8d) ", i+1);
if(!test(should_pass_strings[i].str, should_pass_strings[i].flags)){
if(!fun(should_pass_strings[i].str, should_pass_strings[i].flags)){
++passed;
}else{
fprintf(stderr, "%13s%s\n", "", should_pass_strings[i].str);
@@ -145,11 +182,21 @@ int main(){
printf("Running %d tests that should fail...\n", should_fail_cnt);
for(unsigned i = 0;i < sizeof(should_fail_strings)/sizeof(should_fail_strings[0]);++i){
fprintf(stderr, "%8d) ", i+1);
if(test(should_fail_strings[i].str, should_fail_strings[i].flags)){
if(fun(should_fail_strings[i].str, should_fail_strings[i].flags)){
++passed;
}else{
fprintf(stderr, "%13s%s\n", "", should_fail_strings[i].str);
}
}
fprintf(stderr, "\nResults: %d/%d tests passed\n", passed, total_tests);
return passed;
}
int main(){
int normal_passed = run_test(test);
int cback_passed = run_test(test_cbacks);
int total_passed = normal_passed + cback_passed;
fprintf(stderr, "\nResults: %d/%d normal tests passed\n", normal_passed, total_tests);
fprintf(stderr, "Results: %d/%d callback tests passed\n", cback_passed, total_tests);
fprintf(stderr, "Results: %d/%d tests passed\n", total_passed, total_tests*2);
}