Skip to content

Commit 97b1bf9

Browse files
peterzhu2118k0kubun
peterzhu2118
and
k0kubun
authored
[Bug #20270] Fix --parser=prism (#10970)
Co-authored-by: Takashi Kokubun <takashikkbn@gmail.com>
1 parent d1869cf commit 97b1bf9

File tree

2 files changed

+130
-127
lines changed

2 files changed

+130
-127
lines changed

‎ruby.c

Lines changed: 125 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,6 @@ enum feature_flag_bits {
153153
SEP \
154154
X(parsetree_with_comment) \
155155
SEP \
156-
X(prism_parsetree) \
157-
SEP \
158156
X(insns) \
159157
SEP \
160158
X(insns_without_opt) \
@@ -168,7 +166,7 @@ enum dump_flag_bits {
168166
DUMP_BIT(parsetree_with_comment)),
169167
dump_exit_bits = (DUMP_BIT(yydebug) | DUMP_BIT(syntax) |
170168
DUMP_BIT(parsetree) | DUMP_BIT(parsetree_with_comment) |
171-
DUMP_BIT(prism_parsetree) | DUMP_BIT(insns) | DUMP_BIT(insns_without_opt))
169+
DUMP_BIT(insns) | DUMP_BIT(insns_without_opt))
172170
};
173171

174172
static inline void
@@ -355,7 +353,7 @@ usage(const char *name, int help, int highlight, int columns)
355353

356354
static const struct ruby_opt_message help_msg[] = {
357355
M("--copyright", "", "print the copyright"),
358-
M("--dump={insns|parsetree|prism_parsetree|...}[,...]", "",
356+
M("--dump={insns|parsetree|...}[,...]", "",
359357
"dump debug information. see below for available dump list"),
360358
M("--enable={jit|rubyopt|...}[,...]", ", --disable={jit|rubyopt|...}[,...]",
361359
"enable or disable features. see below for available features"),
@@ -1986,12 +1984,96 @@ env_var_truthy(const char *name)
19861984

19871985
rb_pid_t rb_fork_ruby(int *status);
19881986

1987+
static rb_ast_t *
1988+
process_script(ruby_cmdline_options_t *opt)
1989+
{
1990+
rb_ast_t *ast;
1991+
VALUE parser = rb_parser_new();
1992+
1993+
if (opt->dump & DUMP_BIT(yydebug)) {
1994+
rb_parser_set_yydebug(parser, Qtrue);
1995+
}
1996+
1997+
if (opt->dump & DUMP_BIT(error_tolerant)) {
1998+
rb_parser_error_tolerant(parser);
1999+
}
2000+
2001+
if (opt->e_script) {
2002+
VALUE progname = rb_progname;
2003+
rb_parser_set_context(parser, 0, TRUE);
2004+
2005+
ruby_opt_init(opt);
2006+
ruby_set_script_name(progname);
2007+
rb_parser_set_options(parser, opt->do_print, opt->do_loop,
2008+
opt->do_line, opt->do_split);
2009+
ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
2010+
}
2011+
else {
2012+
VALUE f;
2013+
int xflag = opt->xflag;
2014+
f = open_load_file(opt->script_name, &xflag);
2015+
opt->xflag = xflag != 0;
2016+
rb_parser_set_context(parser, 0, f == rb_stdin);
2017+
ast = load_file(parser, opt->script_name, f, 1, opt);
2018+
}
2019+
if (!ast->body.root) {
2020+
rb_ast_dispose(ast);
2021+
return NULL;
2022+
}
2023+
return ast;
2024+
}
2025+
2026+
static void
2027+
prism_script(ruby_cmdline_options_t *opt, pm_string_t *input, pm_options_t *options)
2028+
{
2029+
ruby_opt_init(opt);
2030+
2031+
if (strcmp(opt->script, "-") == 0) {
2032+
rb_warn("Prism support for streaming code from stdin is not currently supported");
2033+
pm_string_constant_init(input, "", 0);
2034+
pm_options_filepath_set(options, "-e");
2035+
}
2036+
else if (opt->e_script) {
2037+
pm_string_constant_init(input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
2038+
pm_options_filepath_set(options, "-e");
2039+
}
2040+
else {
2041+
pm_string_mapped_init(input, RSTRING_PTR(opt->script_name));
2042+
pm_options_filepath_set(options, RSTRING_PTR(opt->script_name));
2043+
}
2044+
}
2045+
2046+
static VALUE
2047+
prism_dump_tree(pm_string_t *input, pm_options_t *options)
2048+
{
2049+
pm_parser_t parser;
2050+
pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
2051+
2052+
pm_node_t *node = pm_parse(&parser);
2053+
2054+
pm_buffer_t output_buffer = { 0 };
2055+
2056+
pm_prettyprint(&output_buffer, &parser, node);
2057+
2058+
VALUE tree = rb_str_new(output_buffer.value, output_buffer.length);
2059+
2060+
pm_buffer_free(&output_buffer);
2061+
pm_node_destroy(&parser, node);
2062+
pm_parser_free(&parser);
2063+
2064+
return tree;
2065+
}
2066+
19892067
static VALUE
19902068
process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
19912069
{
1992-
rb_ast_t *ast = 0;
1993-
VALUE parser;
1994-
VALUE script_name;
2070+
rb_ast_t *ast = NULL;
2071+
pm_string_t pm_input = { 0 };
2072+
pm_options_t pm_options = { 0 };
2073+
2074+
#define dispose_result() \
2075+
(ast ? rb_ast_dispose(ast) : (pm_string_free(&pm_input), pm_options_free(&pm_options)))
2076+
19952077
const rb_iseq_t *iseq;
19962078
rb_encoding *enc, *lenc;
19972079
#if UTF8_PATH
@@ -2162,13 +2244,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
21622244
lenc = rb_locale_encoding();
21632245
rb_enc_associate(rb_progname, lenc);
21642246
rb_obj_freeze(rb_progname);
2165-
parser = rb_parser_new();
2166-
if (opt->dump & DUMP_BIT(yydebug)) {
2167-
rb_parser_set_yydebug(parser, Qtrue);
2168-
}
2169-
if (opt->dump & DUMP_BIT(error_tolerant)) {
2170-
rb_parser_error_tolerant(parser);
2171-
}
21722247
if (opt->ext.enc.name != 0) {
21732248
opt->ext.enc.index = opt_enc_index(opt->ext.enc.name);
21742249
}
@@ -2194,7 +2269,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
21942269
ienc = enc;
21952270
#endif
21962271
}
2197-
script_name = opt->script_name;
21982272
rb_enc_associate(opt->script_name, IF_UTF8_PATH(uenc, lenc));
21992273
#if UTF8_PATH
22002274
if (uenc != lenc) {
@@ -2261,46 +2335,35 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
22612335
ruby_set_argv(argc, argv);
22622336
opt->sflag = process_sflag(opt->sflag);
22632337

2264-
if (!(*rb_ruby_prism_ptr())) {
2265-
if (opt->e_script) {
2266-
VALUE progname = rb_progname;
2267-
rb_encoding *eenc;
2268-
rb_parser_set_context(parser, 0, TRUE);
2269-
2270-
if (opt->src.enc.index >= 0) {
2271-
eenc = rb_enc_from_index(opt->src.enc.index);
2272-
}
2273-
else {
2274-
eenc = lenc;
2275-
#if UTF8_PATH
2276-
if (ienc) eenc = ienc;
2277-
#endif
2278-
}
2338+
if (opt->e_script) {
2339+
rb_encoding *eenc;
2340+
if (opt->src.enc.index >= 0) {
2341+
eenc = rb_enc_from_index(opt->src.enc.index);
2342+
}
2343+
else {
2344+
eenc = lenc;
22792345
#if UTF8_PATH
2280-
if (eenc != uenc) {
2281-
opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
2282-
}
2346+
if (ienc) eenc = ienc;
22832347
#endif
2284-
rb_enc_associate(opt->e_script, eenc);
2285-
ruby_opt_init(opt);
2286-
ruby_set_script_name(progname);
2287-
rb_parser_set_options(parser, opt->do_print, opt->do_loop,
2288-
opt->do_line, opt->do_split);
2289-
ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
22902348
}
2291-
else {
2292-
VALUE f;
2293-
int xflag = opt->xflag;
2294-
f = open_load_file(script_name, &xflag);
2295-
opt->xflag = xflag != 0;
2296-
rb_parser_set_context(parser, 0, f == rb_stdin);
2297-
ast = load_file(parser, opt->script_name, f, 1, opt);
2349+
#if UTF8_PATH
2350+
if (eenc != uenc) {
2351+
opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
22982352
}
2353+
#endif
2354+
rb_enc_associate(opt->e_script, eenc);
2355+
}
2356+
2357+
if (!(*rb_ruby_prism_ptr())) {
2358+
if (!(ast = process_script(opt))) return Qfalse;
2359+
}
2360+
else {
2361+
prism_script(opt, &pm_input, &pm_options);
22992362
}
23002363
ruby_set_script_name(opt->script_name);
2301-
if (dump & DUMP_BIT(yydebug)) {
2302-
dump &= ~DUMP_BIT(yydebug);
2303-
if (!dump) return Qtrue;
2364+
if ((dump & DUMP_BIT(yydebug)) && !(dump &= ~DUMP_BIT(yydebug))) {
2365+
dispose_result();
2366+
return Qtrue;
23042367
}
23052368

23062369
if (opt->ext.enc.index >= 0) {
@@ -2320,11 +2383,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
23202383
rb_enc_set_default_internal(Qnil);
23212384
rb_stdio_set_default_encoding();
23222385

2323-
if (!(*rb_ruby_prism_ptr()) && !ast->body.root) {
2324-
rb_ast_dispose(ast);
2325-
return Qfalse;
2326-
}
2327-
23282386
opt->sflag = process_sflag(opt->sflag);
23292387
opt->xflag = 0;
23302388

@@ -2341,60 +2399,28 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
23412399
rb_define_global_function("chomp", rb_f_chomp, -1);
23422400
}
23432401

2344-
if (dump & (DUMP_BIT(prism_parsetree))) {
2345-
pm_string_t input;
2346-
pm_options_t options = { 0 };
2347-
2348-
if (strcmp(opt->script, "-") == 0) {
2349-
int xflag = opt->xflag;
2350-
VALUE rb_source = open_load_file(opt->script_name, &xflag);
2351-
opt->xflag = xflag != 0;
2352-
2353-
rb_warn("Prism support for streaming code from stdin is not currently supported");
2354-
pm_string_constant_init(&input, RSTRING_PTR(rb_source), RSTRING_LEN(rb_source));
2355-
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
2356-
}
2357-
else if (opt->e_script) {
2358-
pm_string_constant_init(&input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
2359-
pm_options_filepath_set(&options, "-e");
2402+
if (dump & (DUMP_BIT(parsetree)|DUMP_BIT(parsetree_with_comment))) {
2403+
VALUE tree;
2404+
if (ast) {
2405+
int comment = dump & DUMP_BIT(parsetree_with_comment);
2406+
tree = rb_parser_dump_tree(ast->body.root, comment);
23602407
}
23612408
else {
2362-
pm_string_mapped_init(&input, RSTRING_PTR(opt->script_name));
2363-
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
2409+
tree = prism_dump_tree(&pm_input, &pm_options);
23642410
}
2365-
2366-
pm_parser_t parser;
2367-
pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options);
2368-
2369-
pm_node_t *node = pm_parse(&parser);
2370-
pm_buffer_t output_buffer = { 0 };
2371-
2372-
pm_prettyprint(&output_buffer, &parser, node);
2373-
rb_io_write(rb_stdout, rb_str_new((const char *) output_buffer.value, output_buffer.length));
2374-
rb_io_flush(rb_stdout);
2375-
2376-
pm_buffer_free(&output_buffer);
2377-
pm_node_destroy(&parser, node);
2378-
pm_parser_free(&parser);
2379-
2380-
pm_string_free(&input);
2381-
pm_options_free(&options);
2382-
}
2383-
2384-
if (dump & (DUMP_BIT(parsetree)|DUMP_BIT(parsetree_with_comment))) {
2385-
rb_io_write(rb_stdout, rb_parser_dump_tree(ast->body.root, dump & DUMP_BIT(parsetree_with_comment)));
2411+
rb_io_write(rb_stdout, tree);
23862412
rb_io_flush(rb_stdout);
23872413
dump &= ~DUMP_BIT(parsetree)&~DUMP_BIT(parsetree_with_comment);
23882414
if (!dump) {
2389-
rb_ast_dispose(ast);
2415+
dispose_result();
23902416
return Qtrue;
23912417
}
23922418
}
23932419

23942420
{
23952421
VALUE path = Qnil;
23962422
if (!opt->e_script && strcmp(opt->script, "-")) {
2397-
path = rb_realpath_internal(Qnil, script_name, 1);
2423+
path = rb_realpath_internal(Qnil, opt->script_name, 1);
23982424
#if UTF8_PATH
23992425
if (uenc != lenc) {
24002426
path = str_conv_enc(path, uenc, lenc);
@@ -2405,41 +2431,17 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
24052431
}
24062432
}
24072433

2434+
bool optimize = !(dump & DUMP_BIT(insns_without_opt));
24082435

2409-
if ((*rb_ruby_prism_ptr())) {
2410-
pm_string_t input;
2411-
pm_options_t options = { 0 };
2412-
2413-
if (strcmp(opt->script, "-") == 0) {
2414-
int xflag = opt->xflag;
2415-
VALUE rb_source = open_load_file(opt->script_name, &xflag);
2416-
opt->xflag = xflag != 0;
2417-
2418-
rb_warn("Prism support for streaming code from stdin is not currently supported");
2419-
pm_string_constant_init(&input, RSTRING_PTR(rb_source), RSTRING_LEN(rb_source));
2420-
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
2421-
}
2422-
else if (opt->e_script) {
2423-
pm_string_constant_init(&input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
2424-
pm_options_filepath_set(&options, "-e");
2425-
}
2426-
else {
2427-
pm_string_mapped_init(&input, RSTRING_PTR(opt->script_name));
2428-
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
2429-
}
2430-
2431-
iseq = rb_iseq_new_main_prism(&input, &options, path);
2432-
ruby_opt_init(opt);
2433-
2434-
pm_string_free(&input);
2435-
pm_options_free(&options);
2436+
if (!ast) {
2437+
iseq = rb_iseq_new_main_prism(&pm_input, &pm_options, path);
24362438
}
24372439
else {
24382440
rb_binding_t *toplevel_binding;
24392441
GetBindingPtr(rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING")),
24402442
toplevel_binding);
24412443
const struct rb_block *base_block = toplevel_context(toplevel_binding);
2442-
iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), !(dump & DUMP_BIT(insns_without_opt)));
2444+
iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), optimize);
24432445
rb_ast_dispose(ast);
24442446
}
24452447
}

‎test/ruby/test_rubyoptions.rb

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -287,17 +287,17 @@ def test_rjit_version
287287
end
288288
end
289289

290-
def test_parser_flag
291-
warning = /compiler based on the Prism parser is currently experimental/
290+
PRISM_WARNING = /compiler based on the Prism parser is currently experimental/
292291

293-
assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), warning)
292+
def test_parser_flag
293+
assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), PRISM_WARNING)
294294

295295
assert_in_out_err(%w(--parser=parse.y -e) + ["puts :hi"], "", %w(hi), [])
296296
assert_norun_with_rflag('--parser=parse.y', '--version', "")
297297

298298
assert_in_out_err(%w(--parser=notreal -e) + ["puts :hi"], "", [], /unknown parser notreal/)
299299

300-
assert_in_out_err(%w(--parser=prism --version), "", /\+PRISM/, warning)
300+
assert_in_out_err(%w(--parser=prism --version), "", /\+PRISM/, PRISM_WARNING)
301301
end
302302

303303
def test_eval
@@ -1140,6 +1140,7 @@ def test_dump_parsetree_with_rflag
11401140
assert_norun_with_rflag('--dump=parsetree', '-e', '#frozen-string-literal: true')
11411141
assert_norun_with_rflag('--dump=parsetree+error_tolerant')
11421142
assert_norun_with_rflag('--dump=parse+error_tolerant')
1143+
assert_in_out_err(%w(--parser=prism --dump=parsetree -e ""), "", /ProgramNode/, PRISM_WARNING, encoding: "UTF-8")
11431144
end
11441145

11451146
def test_dump_insns_with_rflag

0 commit comments

Comments
 (0)