Skip to content

Commit 9a19cfd

Browse files
hasumikinyui-knk
hasumikin
authored and
yui-knk
committed
[Universal Parser] Reduce dependence on RArray in parse.y
- Introduce `rb_parser_ary_t` structure to partly eliminate RArray from parse.y - In this patch, `parser_params->tokens` and `parser_params->ast->node_buffer->tokens` are now `rb_parser_ary_t *` - Instead, `ast_node_all_tokens()` internally creates a Ruby Array object from the `rb_parser_ary_t` - Also, delete `rb_ast_tokens()` and `rb_ast_set_tokens()` in node.c - Implement `rb_parser_str_escape()` - This is a port of the `rb_str_escape()` function in string.c - `rb_parser_str_escape()` does not depend on `VALUE` (RString) - Instead, it uses `rb_parser_stirng_t *` - This function works when --dump=y option passed - Because WIP of the universal parser, similar functions like `rb_parser_tokens_free()` exist in both node.c and parse.y. Refactoring them may be needed in some way in the future - Although we considered redesigning the structure: `ast->node_buffer->tokens` into `ast->tokens`, we leave it as it is because `rb_ast_t` is an imemo. (We will address it in the future)
1 parent f42164e commit 9a19cfd

File tree

7 files changed

+429
-215
lines changed

7 files changed

+429
-215
lines changed

‎ast.c

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -774,10 +774,35 @@ ast_node_last_column(rb_execution_context_t *ec, VALUE self)
774774
static VALUE
775775
ast_node_all_tokens(rb_execution_context_t *ec, VALUE self)
776776
{
777+
long i;
777778
struct ASTNodeData *data;
779+
rb_parser_ary_t *parser_tokens;
780+
rb_parser_ast_token_t *parser_token;
781+
VALUE str, loc, token, all_tokens;
782+
778783
TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data);
779784

780-
return rb_ast_tokens(data->ast);
785+
parser_tokens = data->ast->node_buffer->tokens;
786+
if (parser_tokens == NULL) {
787+
return Qnil;
788+
}
789+
790+
all_tokens = rb_ary_new2(parser_tokens->len);
791+
for (i = 0; i < parser_tokens->len; i++) {
792+
parser_token = parser_tokens->data[i];
793+
str = rb_str_new(parser_token->str->ptr, parser_token->str->len);
794+
loc = rb_ary_new_from_args(4,
795+
INT2FIX(parser_token->loc.beg_pos.lineno),
796+
INT2FIX(parser_token->loc.beg_pos.column),
797+
INT2FIX(parser_token->loc.end_pos.lineno),
798+
INT2FIX(parser_token->loc.end_pos.column)
799+
);
800+
token = rb_ary_new_from_args(4, INT2FIX(parser_token->id), ID2SYM(rb_intern(parser_token->type_name)), str, loc);
801+
rb_ary_push(all_tokens, token);
802+
}
803+
rb_obj_freeze(all_tokens);
804+
805+
return all_tokens;
781806
}
782807

783808
static VALUE

‎node.c

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ rb_node_buffer_new(void)
6969
init_node_buffer_list(&nb->unmarkable, (node_buffer_elem_t*)&nb[1], ruby_xmalloc);
7070
init_node_buffer_list(&nb->markable, (node_buffer_elem_t*)((size_t)nb->unmarkable.head + bucket_size), ruby_xmalloc);
7171
nb->local_tables = 0;
72-
nb->tokens = Qnil;
72+
nb->tokens = 0;
7373
#ifdef UNIVERSAL_PARSER
7474
nb->config = config;
7575
#endif
@@ -176,6 +176,24 @@ parser_string_free(rb_ast_t *ast, rb_parser_string_t *str)
176176
xfree(str);
177177
}
178178

179+
static void
180+
parser_ast_token_free(rb_ast_t *ast, rb_parser_ast_token_t *token)
181+
{
182+
if (!token) return;
183+
parser_string_free(ast, token->str);
184+
xfree(token);
185+
}
186+
187+
static void
188+
parser_tokens_free(rb_ast_t *ast, rb_parser_ary_t *tokens)
189+
{
190+
for (long i = 0; i < tokens->len; i++) {
191+
parser_ast_token_free(ast, tokens->data[i]);
192+
}
193+
xfree(tokens->data);
194+
xfree(tokens);
195+
}
196+
179197
static void
180198
free_ast_value(rb_ast_t *ast, void *ctx, NODE *node)
181199
{
@@ -228,6 +246,9 @@ free_ast_value(rb_ast_t *ast, void *ctx, NODE *node)
228246
static void
229247
rb_node_buffer_free(rb_ast_t *ast, node_buffer_t *nb)
230248
{
249+
if (ast->node_buffer && ast->node_buffer->tokens) {
250+
parser_tokens_free(ast, ast->node_buffer->tokens);
251+
}
231252
iterate_node_values(ast, &nb->unmarkable, free_ast_value, NULL);
232253
node_buffer_list_free(ast, &nb->unmarkable);
233254
node_buffer_list_free(ast, &nb->markable);
@@ -388,8 +409,6 @@ void
388409
rb_ast_mark_and_move(rb_ast_t *ast, bool reference_updating)
389410
{
390411
if (ast->node_buffer) {
391-
rb_gc_mark_and_move(&ast->node_buffer->tokens);
392-
393412
node_buffer_t *nb = ast->node_buffer;
394413
iterate_node_values(ast, &nb->markable, mark_and_move_ast_value, NULL);
395414

@@ -438,18 +457,6 @@ rb_ast_dispose(rb_ast_t *ast)
438457
rb_ast_free(ast);
439458
}
440459

441-
VALUE
442-
rb_ast_tokens(rb_ast_t *ast)
443-
{
444-
return ast->node_buffer->tokens;
445-
}
446-
447-
void
448-
rb_ast_set_tokens(rb_ast_t *ast, VALUE tokens)
449-
{
450-
RB_OBJ_WRITE(ast, &ast->node_buffer->tokens, tokens);
451-
}
452-
453460
VALUE
454461
rb_node_set_type(NODE *n, enum node_type t)
455462
{

‎node.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ struct node_buffer_struct {
4040
// - text of token
4141
// - location info
4242
// Array, whose entry is array
43-
VALUE tokens;
43+
rb_parser_ary_t *tokens;
4444
#ifdef UNIVERSAL_PARSER
4545
const rb_parser_config_t *config;
4646
#endif
@@ -55,7 +55,6 @@ rb_ast_t *rb_ast_new(void);
5555
#endif
5656
size_t rb_ast_memsize(const rb_ast_t*);
5757
void rb_ast_dispose(rb_ast_t*);
58-
VALUE rb_ast_tokens(rb_ast_t *ast);
5958
#if RUBY_DEBUG
6059
void rb_ast_node_type_change(NODE *n, enum node_type type);
6160
#endif
@@ -65,7 +64,6 @@ void rb_node_init(NODE *n, enum node_type type);
6564
void rb_ast_mark_and_move(rb_ast_t *ast, bool reference_updating);
6665
void rb_ast_update_references(rb_ast_t*);
6766
void rb_ast_free(rb_ast_t*);
68-
void rb_ast_set_tokens(rb_ast_t*, VALUE);
6967
NODE *rb_ast_newnode(rb_ast_t*, enum node_type type, size_t size, size_t alignment);
7068
void rb_ast_delete_node(rb_ast_t*, NODE *n);
7169
rb_ast_id_table_t *rb_ast_new_local_table(rb_ast_t*, int);

0 commit comments

Comments
 (0)