From 900f25f8322b401a7be2c5fde9fc950aa7c85108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 29 Oct 2023 12:30:32 +0100 Subject: [PATCH 001/185] Branch out v1.16 --- Makefile | 2 +- SECURITY.md | 5 ++--- .../pages/references/compatibility-and-deprecations.md | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 1dd0b18fdc2..c885b10be4f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PREFIX ?= /usr/local TEST_FILES ?= "*_test.exs" SHARE_PREFIX ?= $(PREFIX)/share MAN_PREFIX ?= $(SHARE_PREFIX)/man -CANONICAL := main/ +# CANONICAL := main/ ELIXIRC := bin/elixirc --ignore-module-conflict $(ELIXIRC_OPTS) ERLC := erlc -I lib/elixir/include ERL_MAKE := if [ -n "$(ERLC_OPTS)" ]; then ERL_COMPILER_OPTIONS=$(ERLC_OPTS) erl -make; else erl -make; fi diff --git a/SECURITY.md b/SECURITY.md index 69252d3ad7c..be691f8303a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,12 +6,11 @@ Elixir applies bug fixes only to the latest minor branch. Security patches are a Elixir version | Support :------------- | :----------------------------- -1.16 | Development -1.15 | Bug fixes and security patches +1.16 | Bug fixes and security patches +1.15 | Security patches only 1.14 | Security patches only 1.13 | Security patches only 1.12 | Security patches only -1.11 | Security patches only ## Announcements diff --git a/lib/elixir/pages/references/compatibility-and-deprecations.md b/lib/elixir/pages/references/compatibility-and-deprecations.md index 9159ae16209..48441356ce7 100644 --- a/lib/elixir/pages/references/compatibility-and-deprecations.md +++ b/lib/elixir/pages/references/compatibility-and-deprecations.md @@ -8,12 +8,11 @@ Elixir applies bug fixes only to the latest minor branch. Security patches are a Elixir version | Support :------------- | :----------------------------- -1.16 | Development -1.15 | Bug fixes and security patches +1.16 | Bug fixes and security patches +1.15 | Security patches only 1.14 | Security patches only 1.13 | Security patches only 1.12 | Security patches only -1.11 | Security patches only New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann). All security releases [will be tagged with `[security]`](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). @@ -43,6 +42,7 @@ Erlang/OTP versioning is independent from the versioning of Elixir. Erlang relea Elixir version | Supported Erlang/OTP versions :------------- | :------------------------------- +1.16 | 24 - 26 1.15 | 24 - 26 1.14 | 23 - 25 1.13 | 22 - 24 (and Erlang/OTP 25 from v1.13.4) From 968b51319ee84f5dea1a4c5b5d4a06674269891d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 30 Oct 2023 08:11:18 +0100 Subject: [PATCH 002/185] Clarify scope of anti-patterns --- lib/elixir/pages/anti-patterns/what-anti-patterns.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/what-anti-patterns.md b/lib/elixir/pages/anti-patterns/what-anti-patterns.md index 41c7cef5090..b33e69f7dff 100644 --- a/lib/elixir/pages/anti-patterns/what-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/what-anti-patterns.md @@ -4,9 +4,11 @@ Anti-patterns describe common mistakes or indicators of potential problems in co They are also known as "code smells". The goal of these guides is to document known anti-patterns found in Elixir software -and teach developers how to identify and correct them. If an existing piece of code -matches an anti-pattern, it does not mean your code must be rewritten. However, you -should take its potential pitfalls and alternatives into consideration. +and teach developers how to identify them and their limitations. If an existing piece +of code matches an anti-pattern, it does not mean your code must be rewritten. +No codebase is free of anti-patterns and one should not aim to remove all +anti-patterns of a codebase. The goal is to promote a discussion of potential +pitfalls and provide alternatives into consideration. The anti-patterns in these guides are broken into 4 main categories: From 8bbba572dec29f6f924db0d0ed29fe071d899829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 30 Oct 2023 09:18:18 +0100 Subject: [PATCH 003/185] Describe pattern matching as simpler --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index 07004922b24..093b1a60b04 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -436,7 +436,7 @@ iex> Graphics.plot(point_3d) Overall, the usage of `map.key` and `map[:key]` encode important information about your data structure, allowing developers to be clear about their intent. See both `Map` and `Access` module documentation for more information and examples. -Another alternative to refactor this anti-pattern is to use pattern matching: +An even simpler alternative to refactor this anti-pattern is to use pattern matching: ```elixir defmodule Graphics do @@ -461,6 +461,8 @@ iex> Graphics.plot(point_3d) {5, 6, nil} ``` +Pattern-matching is specially useful when matching over multiple keys at once and also when you want to match and assert on the values of a map. + Another alternative is to use structs. By default, structs only support static access to its fields: ```elixir From 3036401c7c25dcc47f1317e7c0dccbee2a744f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 30 Oct 2023 15:12:28 +0100 Subject: [PATCH 004/185] Clarify best practices and update anti-patterns list --- .../pages/anti-patterns/code-anti-patterns.md | 72 +------------------ .../pages/anti-patterns/what-anti-patterns.md | 2 +- .../alias-require-and-import.md | 2 +- 3 files changed, 4 insertions(+), 72 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index 093b1a60b04..fa5bd56d633 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -34,7 +34,8 @@ You could refactor the code above like this: @five_min_in_seconds 60 * 5 defp unix_five_min_from_now do - unix_now = DateTime.to_unix(DateTime.utc_now(), :second) + now = DateTime.utc_now() + unix_now = DateTime.to_unix(now, :second) unix_now + @five_min_in_seconds end ``` @@ -45,75 +46,6 @@ We removed the unnecessary comments. We also added a `@five_min_in_seconds` modu Elixir makes a clear distinction between **documentation** and code comments. The language has built-in first-class support for documentation through `@doc`, `@moduledoc`, and more. See the ["Writing documentation"](../getting-started/writing-documentation.md) guide for more information. -## Complex branching - -#### Problem - -When a function assumes the responsibility of handling multiple errors alone, it can increase its cyclomatic complexity (metric of control-flow) and become incomprehensible. This situation can configure a specific instance of "Long function", a traditional anti-pattern, but has implications of its own. Under these circumstances, this function could get very confusing, difficult to maintain and test, and therefore bug-proneness. - -#### Example - -An example of this anti-pattern is when a function uses the `case` control-flow structure or other similar constructs (for example, `cond` or `receive`) to handle variations of a return type. This practice can make the function more complex, long, and difficult to understand, as shown next. - -```elixir -def get_customer(customer_id) do - case SomeHTTPClient.get("/customers/#{customer_id}") do - {:ok, %{status: 200, body: body}} -> - case Jason.decode(body) do - {:ok, decoded} -> - %{ - "first_name" => first_name, - "last_name" => last_name, - "company" => company - } = decoded - - customer = - %Customer{ - id: customer_id, - name: "#{first_name} #{last_name}", - company: company - } - - {:ok, customer} - - {:error, _} -> - {:error, "invalid response body"} - end - - {:error, %{status: status, body: body}} -> - case Jason.decode(body) do - %{"error" => message} when is_binary(message) -> - {:error, message} - - %{} -> - {:error, "invalid response with status #{status}"} - end - end -end -``` - -The code above is complex because the `case` clauses are long and often have their own branching logic in them. With the clauses spread out, it is hard to understand what each clause does individually and it is hard to see all of the different scenarios your code pattern matches on. - -#### Refactoring - -As shown below, in this situation, instead of concentrating all error handling within the same function, creating complex branches, it is better to delegate each branch to a different private function. In this way, the code will be cleaner and more readable. - -```elixir -def get_customer(customer_id) do - case SomeHTTPClient.get("/customers/#{customer_id}") do - {:ok, %{status: 200, body: body}} -> - http_customer_to_struct(customer_id, body) - - {:error, %{status: status, body: body}} -> - http_error(status, body) - end -end -``` - -Both `http_customer_to_struct(customer_id, body)` and `http_error(status, body)` above contain the previous branches refactored into private functions. - -It is worth noting that this refactoring is trivial to perform in Elixir because clauses cannot define variables or otherwise affect their parent scope. Therefore, extracting any clause or branch to a private function is a matter of gathering all variables used in that branch and passing them as arguments to the new function. - ## Complex `else` clauses in `with` #### Problem diff --git a/lib/elixir/pages/anti-patterns/what-anti-patterns.md b/lib/elixir/pages/anti-patterns/what-anti-patterns.md index b33e69f7dff..1cb4e72eb65 100644 --- a/lib/elixir/pages/anti-patterns/what-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/what-anti-patterns.md @@ -38,4 +38,4 @@ Each anti-pattern is documented using the following structure: The initial catalog of anti-patterns was proposed by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/). For more info, see [Understanding Code Smells in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf) and [the associated code repository](https://github.com/lucasvegi/Elixir-Code-Smells). -Additionally, the Security Working Group of the [Erlang Ecosystem Foundation](https://erlef.github.io/security-wg/) publishes [documents with security resources and best-practices of both Erland and Elixir, including detailed guides for web applications](https://erlef.github.io/security-wg/). +Additionally, the Security Working Group of the [Erlang Ecosystem Foundation](https://erlef.github.io/security-wg/) publishes [documents with security resources and best-practices of both Erlang and Elixir, including detailed guides for web applications](https://erlef.github.io/security-wg/). diff --git a/lib/elixir/pages/getting-started/alias-require-and-import.md b/lib/elixir/pages/getting-started/alias-require-and-import.md index c3be2418837..2e45feef2e5 100644 --- a/lib/elixir/pages/getting-started/alias-require-and-import.md +++ b/lib/elixir/pages/getting-started/alias-require-and-import.md @@ -112,7 +112,7 @@ end In the example above, the imported `List.duplicate/2` is only visible within that specific function. `duplicate/2` won't be available in any other function in that module (or any other module for that matter). -Note that `import`s are generally discouraged in the language. When working on your own code, prefer `alias` to `import`. +While `import`s can be a useful for frameworks and libraries to build abstractions, developers should generally prefer `alias` to `import` on their own codebases, as aliases make the origin of the function being invoked clearer. ## use From 4b795871b6a2e32f4c3a160f00f2882f531a5b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 30 Oct 2023 15:50:36 +0100 Subject: [PATCH 005/185] Improve examples and docs --- lib/elixir/lib/registry.ex | 14 ++-- .../pages/anti-patterns/code-anti-patterns.md | 2 +- .../anti-patterns/design-anti-patterns.md | 82 +++++-------------- 3 files changed, 30 insertions(+), 68 deletions(-) diff --git a/lib/elixir/lib/registry.ex b/lib/elixir/lib/registry.ex index cde2af610a0..4177e35bae7 100644 --- a/lib/elixir/lib/registry.ex +++ b/lib/elixir/lib/registry.ex @@ -27,8 +27,8 @@ defmodule Registry do `Registry.start_link/1`, it can be used to register and access named processes using the `{:via, Registry, {registry, key}}` tuple: - {:ok, _} = Registry.start_link(keys: :unique, name: Registry.ViaTest) - name = {:via, Registry, {Registry.ViaTest, "agent"}} + {:ok, _} = Registry.start_link(keys: :unique, name: MyApp.Registry) + name = {:via, Registry, {MyApp.Registry, "agent"}} {:ok, _} = Agent.start_link(fn -> 0 end, name: name) Agent.get(name, & &1) #=> 0 @@ -39,22 +39,22 @@ defmodule Registry do In the previous example, we were not interested in associating a value to the process: - Registry.lookup(Registry.ViaTest, "agent") + Registry.lookup(MyApp.Registry, "agent") #=> [{self(), nil}] However, in some cases it may be desired to associate a value to the process using the alternate `{:via, Registry, {registry, key, value}}` tuple: - {:ok, _} = Registry.start_link(keys: :unique, name: Registry.ViaTest) - name = {:via, Registry, {Registry.ViaTest, "agent", :hello}} + {:ok, _} = Registry.start_link(keys: :unique, name: MyApp.Registry) + name = {:via, Registry, {MyApp.Registry, "agent", :hello}} {:ok, agent_pid} = Agent.start_link(fn -> 0 end, name: name) - Registry.lookup(Registry.ViaTest, "agent") + Registry.lookup(MyApp.Registry, "agent") #=> [{agent_pid, :hello}] To this point, we have been starting `Registry` using `start_link/1`. Typically the registry is started as part of a supervision tree though: - {Registry, keys: :unique, name: Registry.ViaTest} + {Registry, keys: :unique, name: MyApp.Registry} Only registries with unique keys can be used in `:via`. If the name is already taken, the case-specific `start_link` function (`Agent.start_link/2` diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index fa5bd56d633..6ae84a3db5a 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -299,7 +299,7 @@ There are few known exceptions to this anti-pattern: * [Protocol implementations](`Kernel.defimpl/2`) are, by design, defined under the protocol namespace - * [Custom Mix tasks](`Mix.Task`) are always defined under the `Mix.Tasks` namespace, such as `Mix.Tasks.PlugAuth` + * In some scenarios, the namespace owner may allow exceptions to this rule. For example, in Elixir itself, you defined [custom Mix tasks](`Mix.Task`) by placing them under the `Mix.Tasks` namespace, such as `Mix.Tasks.PlugAuth` * If you are the maintainer for both `plug` and `plug_auth`, then you may allow `plug_auth` to define modules with the `Plug` namespace, such as `Plug.Auth`. However, you are responsible for avoiding or managing any conflicts that may arise in the future diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index 8a1b13d562b..7ec9f992147 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -126,7 +126,7 @@ Remember booleans are internally represented as atoms. Therefore there is no per #### Problem -This anti-pattern refers to code that uses exceptions for control flow. Exception handling itself does not represent an anti-pattern, but developers must prefer to use `case` and pattern matching to change the flow of their code, instead of `try/rescue`. In turn, library authors should provide developers with APIs to handle errors without relying on exception handling. When developers have no freedom to decide if an error is exceptional or not, this is considered an anti-pattern. +This anti-pattern refers to code that uses `Exception`s for control flow. Exception handling itself does not represent an anti-pattern, but developers must prefer to use `case` and pattern matching to change the flow of their code, instead of `try/rescue`. In turn, library authors should provide developers with APIs to handle errors without relying on exception handling. When developers have no freedom to decide if an error is exceptional or not, this is considered an anti-pattern. #### Example @@ -186,63 +186,11 @@ end A common practice followed by the community is to make the non-raising version return `{:ok, result}` or `{:error, Exception.t}`. For example, an HTTP client may return `{:ok, %HTTP.Response{}}` on success cases and `{:error, %HTTP.Error{}}` for failures, where `HTTP.Error` is [implemented as an exception](`Kernel.defexception/1`). This makes it convenient for anyone to raise an exception by simply calling `Kernel.raise/1`. -## Feature envy - -#### Problem - -This anti-pattern occurs when a function accesses more data or calls more functions from another module than from its own. The presence of this anti-pattern can make a module less cohesive and increase code coupling. - -#### Example - -In the following code, all the data used in the `calculate_total_item/1` function of the module `Order` comes from the `OrderItem` module. This increases coupling and decreases code cohesion unnecessarily. - -```elixir -defmodule Order do - # Some functions... - - def calculate_total_item(id) do - item = OrderItem.find_item(id) - total = (item.price + item.taxes) * item.amount - - if discount = OrderItem.find_discount(item) do - total - total * discount - else - total - end - end -end -``` - -#### Refactoring - -To remove this anti-pattern we can move `calculate_total_item/1` to `OrderItem`, decreasing coupling: - -```elixir -defmodule OrderItem do - def find_item(id) - def find_discount(item) - - def calculate_total_item(id) do # <= function moved from Order! - item = find_item(id) - total = (item.price + item.taxes) * item.amount - discount = find_discount(item) - - unless is_nil(discount) do - total - total * discount - else - total - end - end -end -``` - -This refactoring is only possible when you own both modules. If the module you are invoking belongs to another application, then it is not possible to add new functions to it, and your only option is to define an additional module that augments the third-party module. - ## Primitive obsession #### Problem -This anti-pattern happens when Elixir basic types (for example, *integer*, *float*, and *string*) are abusively used in function parameters and code variables, rather than creating specific composite data types (for example, *tuples*, *maps*, and *structs*) that can better represent a domain. +This anti-pattern happens when Elixir basic types (for example, *integer*, *float*, and *string*) are excessively used to carry structured information, rather than creating specific composite data types (for example, *tuples*, *maps*, and *structs*) that can better represent a domain. #### Example @@ -250,17 +198,23 @@ An example of this anti-pattern is the use of a single *string* to represent an ```elixir defmodule MyApp do - def process_address(address) when is_binary(address) do - # Do something with address... + def extract_postal_code(address) when is_binary(address) do + # Extract postal code with address... + end + + def fill_in_country(address) when is_binary(address) do + # Fill in missing country... end end ``` +While you may receive the `address` as a string from a database, web request, or a third-party, if you find yourself frequently manipulating or extracting information from the string, it is a good indicator you should convert the address into structured data: + Another example of this anti-pattern is using floating numbers to model money and currency, when [richer data structures should be preferred](https://hexdocs.pm/ex_money/). #### Refactoring -Possible solutions to this anti-pattern is to use maps or structs to model our address. The example below creates an `Address` struct, better representing this domain through a composite type. Additionally, we can modify the `process_address/1` function to accept a parameter of type `Address` instead of a *string*. With this modification, we can extract each field of this composite type individually when needed. +Possible solutions to this anti-pattern is to use maps or structs to model our address. The example below creates an `Address` struct, better representing this domain through a composite type. Additionally, we introduce a `parse/1` function, that converts the string into an `Address`, which will simplify the logic of remainng functions. With this modification, we can extract each field of this composite type individually when needed. ```elixir defmodule Address do @@ -270,8 +224,16 @@ end ```elixir defmodule MyApp do - def process_address(%Address{} = address) do - # Do something with address... + def parse(address) when is_binary(address) do + # Returns %Address{} + end + + def extract_postal_code(%Address{} = address) do + # Extract postal code with address... + end + + def fill_in_country(%Address{} = address) do + # Fill in missing country... end end ``` @@ -342,7 +304,7 @@ Using multi-clause functions in Elixir, to group functions of the same name, is A frequent example of this usage of multi-clause functions is when developers mix unrelated business logic into the same function definition. Such functions often have generic names or too broad specifications, making it difficult for maintainers and users of said functions to maintain and understand them. -Some developers may use documentation mechanisms such as `@doc` annotations to compensate for poor code readability, however the documentation itself may end-up full of conditionals to describe how the function behaves for each different argument combination. +Some developers may use documentation mechanisms such as `@doc` annotations to compensate for poor code readability, however the documentation itself may end-up full of conditionals to describe how the function behaves for each different argument combination. This is a good indicator that the clauses are ultimately unrelated. ```elixir @doc """ From 1a8bad1bf238eee4ff4e4f27ae172b51c1c39cfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 30 Oct 2023 20:10:43 +0100 Subject: [PATCH 006/185] Streamline unrelated introduction --- lib/elixir/pages/anti-patterns/design-anti-patterns.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index 7ec9f992147..9afada4bea1 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -298,11 +298,11 @@ The following arguments were given to MyLibrary.foo/1: #### Problem -Using multi-clause functions in Elixir, to group functions of the same name, is not an anti-pattern in itself. However, due to the great flexibility provided by this programming feature, some developers may abuse the number of guard clauses and pattern matches to group *unrelated* functionality. +Using multi-clause functions in Elixir, to group functions of the same name, is a powerful Elixir feature. However, some developers may abuse this feature to group *unrelated* functionality, which configures an anti-pattern. #### Example -A frequent example of this usage of multi-clause functions is when developers mix unrelated business logic into the same function definition. Such functions often have generic names or too broad specifications, making it difficult for maintainers and users of said functions to maintain and understand them. +A frequent example of this usage of multi-clause functions is when developers mix unrelated business logic into the same function definition. Such functions often have generic names or too broad specifications, making it difficult for other developers to understand and maintain them. Some developers may use documentation mechanisms such as `@doc` annotations to compensate for poor code readability, however the documentation itself may end-up full of conditionals to describe how the function behaves for each different argument combination. This is a good indicator that the clauses are ultimately unrelated. From 5c8f9aac647159f0b443a2992d2b13c1764e2e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 30 Oct 2023 23:39:43 +0100 Subject: [PATCH 007/185] Fix getting started links --- lib/elixir/lib/gen_server.ex | 2 +- lib/elixir/lib/process.ex | 2 +- lib/elixir/pages/references/syntax-reference.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/elixir/lib/gen_server.ex b/lib/elixir/lib/gen_server.ex index 79549a9c730..cf581bd5221 100644 --- a/lib/elixir/lib/gen_server.ex +++ b/lib/elixir/lib/gen_server.ex @@ -475,7 +475,7 @@ defmodule GenServer do guide provides a tutorial-like introduction. The documentation and links in Erlang can also provide extra insight. - * [GenServer - Elixir's Getting Started Guide](https://elixir-lang.org/getting-started/mix-otp/genserver.html) + * [GenServer - Elixir's Getting Started Guide](genservers.md) * [`:gen_server` module documentation](`:gen_server`) * [gen_server Behaviour - OTP Design Principles](https://www.erlang.org/doc/design_principles/gen_server_concepts.html) * [Clients and Servers - Learn You Some Erlang for Great Good!](http://learnyousomeerlang.com/clients-and-servers) diff --git a/lib/elixir/lib/process.ex b/lib/elixir/lib/process.ex index e1968538ac2..abb69426a84 100644 --- a/lib/elixir/lib/process.ex +++ b/lib/elixir/lib/process.ex @@ -504,7 +504,7 @@ defmodule Process do If the process is already dead when calling `Process.monitor/1`, a `:DOWN` message is delivered immediately. - See ["The need for monitoring"](https://elixir-lang.org/getting-started/mix-otp/genserver.html#the-need-for-monitoring) + See ["The need for monitoring"](genservers.md#the-need-for-monitoring) for an example. See `:erlang.monitor/2` for more information. Inlined by the compiler. diff --git a/lib/elixir/pages/references/syntax-reference.md b/lib/elixir/pages/references/syntax-reference.md index 5564ceb6af7..857fe110700 100644 --- a/lib/elixir/pages/references/syntax-reference.md +++ b/lib/elixir/pages/references/syntax-reference.md @@ -395,7 +395,7 @@ end All of the constructs above are part of Elixir's syntax and have their own representation as part of the Elixir AST. This section will discuss the remaining constructs that are alternative representations of the constructs above. In other words, the constructs below can be represented in more than one way in your Elixir code and retain AST equivalence. We call this "Optional Syntax". -For a lightweight introduction to Elixir's Optional Syntax, [see this document](https://elixir-lang.org/getting-started/optional-syntax.html). Below we continue with a more complete reference. +For a lightweight introduction to Elixir's Optional Syntax, [see this document](optional-syntax.md). Below we continue with a more complete reference. ### Integers in other bases and Unicode code points From 82818f7126d1f190c590505d42418c3fdc278258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 31 Oct 2023 07:49:22 +0100 Subject: [PATCH 008/185] Improve complex extraction example --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index 6ae84a3db5a..c41dc17e442 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -101,11 +101,11 @@ end #### Problem -When we use multi-clause functions, it is possible to extract values in the clauses for further usage and for pattern matching/guard checking. This extraction itself does not represent an anti-pattern, but when you have too many clauses or too many arguments, it becomes hard to know which extracted parts are used for pattern/guards and what is used only inside the function body. This anti-pattern is related to [Unrelated multi-clause function](design-anti-patterns.md#unrelated-multi-clause-function), but with implications of its own. It impairs the code readability in a different way. +When we use multi-clause functions, it is possible to extract values in the clauses for further usage and for pattern matching/guard checking. This extraction itself does not represent an anti-pattern, but when you have *extractions made across several clauses and several arguments of the same function*, it becomes hard to know which extracted parts are used for pattern/guards and what is used only inside the function body. This anti-pattern is related to [Unrelated multi-clause function](design-anti-patterns.md#unrelated-multi-clause-function), but with implications of its own. It impairs the code readability in a different way. #### Example -The multi-clause function `drive/1` is extracting fields of an `%User{}` struct for usage in the clause expression (`age`) and for usage in the function body (`name`). Ideally, a function should not mix pattern matching extractions for usage in its guard expressions and also in its body. +The multi-clause function `drive/1` is extracting fields of an `%User{}` struct for usage in the clause expression (`age`) and for usage in the function body (`name`): ```elixir def drive(%User{name: name, age: age}) when age >= 18 do @@ -117,7 +117,7 @@ def drive(%User{name: name, age: age}) when age < 18 do end ``` -While the example is small and looks like a clear code, try to imagine a situation where `drive/1` was more complex, having many more clauses, arguments, and extractions. +While the example above is small and does not configure an anti-pattern, it is an example of mixed extraction and pattern matching. A situation where `drive/1` was more complex, having many more clauses, arguments, and extractions, would make it hard to know at a glance which variables are used for pattern/guards and which ones are not. #### Refactoring From f8016bca4895c4f6e3b1927bdf9ed0eb4a400b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 31 Oct 2023 08:52:05 +0100 Subject: [PATCH 009/185] Describe them as potential anti-patterns --- .../pages/anti-patterns/code-anti-patterns.md | 4 +++- .../pages/anti-patterns/design-anti-patterns.md | 2 +- .../pages/anti-patterns/macro-anti-patterns.md | 2 +- .../pages/anti-patterns/process-anti-patterns.md | 2 +- .../pages/anti-patterns/what-anti-patterns.md | 14 ++++++++------ 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index c41dc17e442..22479a3a80b 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -1,6 +1,6 @@ # Code-related anti-patterns -This document outlines anti-patterns related to your code and particular Elixir idioms and features. +This document outlines potential anti-patterns related to your code and particular Elixir idioms and features. ## Comments @@ -295,6 +295,8 @@ defmodule PlugAuth do end ``` +#### Additional remarks + There are few known exceptions to this anti-pattern: * [Protocol implementations](`Kernel.defimpl/2`) are, by design, defined under the protocol namespace diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index 9afada4bea1..f1eb627988a 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -1,6 +1,6 @@ # Design-related anti-patterns -This document outlines anti-patterns related to your modules, functions, and the role they +This document outlines potential anti-patterns related to your modules, functions, and the role they play within a codebase. ## Alternative return types diff --git a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md index 1784c7318e8..26a0c603027 100644 --- a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md @@ -1,6 +1,6 @@ # Meta-programming anti-patterns -This document outlines anti-patterns related to meta-programming. +This document outlines potential anti-patterns related to meta-programming. ## Large code generation by macros diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md index 5d83d8f32fb..7a7378279bc 100644 --- a/lib/elixir/pages/anti-patterns/process-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -1,6 +1,6 @@ # Process-related anti-patterns -This document outlines anti-patterns related to processes and process-based abstractions. +This document outlines potential anti-patterns related to processes and process-based abstractions. ## Code organization by process diff --git a/lib/elixir/pages/anti-patterns/what-anti-patterns.md b/lib/elixir/pages/anti-patterns/what-anti-patterns.md index 1cb4e72eb65..5977eb5464e 100644 --- a/lib/elixir/pages/anti-patterns/what-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/what-anti-patterns.md @@ -1,14 +1,14 @@ # What are anti-patterns? -Anti-patterns describe common mistakes or indicators of potential problems in code. +Anti-patterns describe common mistakes or indicators of problems in code. They are also known as "code smells". -The goal of these guides is to document known anti-patterns found in Elixir software -and teach developers how to identify them and their limitations. If an existing piece +The goal of these guides is to document potential anti-patterns found in Elixir software +and teach developers how to identify them and their pitfalls. If an existing piece of code matches an anti-pattern, it does not mean your code must be rewritten. -No codebase is free of anti-patterns and one should not aim to remove all -anti-patterns of a codebase. The goal is to promote a discussion of potential -pitfalls and provide alternatives into consideration. +Sometimes, even if a snippet matches a potential anti-pattern and its limitations, +it may be the best approach to the problem at hand. No codebase is free of anti-patterns +and one should not aim to remove all of them. The anti-patterns in these guides are broken into 4 main categories: @@ -36,6 +36,8 @@ Each anti-pattern is documented using the following structure: * **Refactoring:** Ways to change your code to improve its qualities. Examples of refactored code are presented to illustrate these changes. +An additional section with "Additional Remarks" may be provided. Those may include known scenarios where the anti-pattern does not apply. + The initial catalog of anti-patterns was proposed by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/). For more info, see [Understanding Code Smells in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf) and [the associated code repository](https://github.com/lucasvegi/Elixir-Code-Smells). Additionally, the Security Working Group of the [Erlang Ecosystem Foundation](https://erlef.github.io/security-wg/) publishes [documents with security resources and best-practices of both Erlang and Elixir, including detailed guides for web applications](https://erlef.github.io/security-wg/). From 792d4cc6310c56eb9772056a6b5fb3339ce17b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 31 Oct 2023 09:16:09 +0100 Subject: [PATCH 010/185] Release v1.16.0-rc.0 --- CHANGELOG.md | 5 +++-- VERSION | 2 +- bin/elixir | 2 +- bin/elixir.bat | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5e1d9b99e8..c728bf541e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,13 +57,13 @@ A huge thank you to Vinícius Muller for working on the new diagnostics. Elixir's Getting Started guided has been made part of the Elixir repository and incorporated into ExDoc. This was an opportunity to revisit and unify all official guides and references. -We have also incorporated and extended the work on [Understanding Code Smells in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf), by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/), into the official document in the form of anti-patterns. The anti-patterns are divided into four categories: code-related, design-related, process-related, and meta-programming. Our goal is to give all developers with both positive and negative examples of Elixir code, with context and examples on how to improve their codebases. +We have also incorporated and extended the work on [Understanding Code Smells in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf), by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/), into the official document in the form of anti-patterns. The anti-patterns are divided into four categories: code-related, design-related, process-related, and meta-programming. Our goal is to give all developers examples of potential anti-patterns, with context and examples on how to improve their codebases. Another [ExDoc](https://github.com/elixir-lang/ex_doc) feature we have incorporated in this release is the addition of cheatsheets, starting with [a cheatsheet for the Enum module](https://hexdocs.pm/elixir/main/enum-cheat.html). If you would like to contribute future cheatsheets to Elixir itself, feel free to start a discussion with an issue. Finally, we have started enriching our documentation with [Mermaid.js](https://mermaid.js.org/) diagrams. You can find examples in the [GenServer](https://hexdocs.pm/elixir/main/GenServer.html) and [Supervisor](https://hexdocs.pm/elixir/main/Supervisor.html) docs. -## v1.16.0-dev +## v1.16.0-rc.0 (2023-10-31) ### 1. Enhancements @@ -76,6 +76,7 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m * [Code] Automatically include columns in parsing options * [Code] Introduce `MismatchedDelimiterError` for handling mismatched delimiter exceptions * [Code.Fragment] Handle anonymous calls in fragments + * [Code.Formatter] Trim trailing whitespace on heredocs with `\r\n` * [Kernel] Suggest module names based on suffix and casing errors when the module does not exist in `UndefinedFunctionError` * [Kernel.ParallelCompiler] Introduce `Kernel.ParallelCompiler.pmap/2` to compile multiple additional entries in parallel * [Kernel.SpecialForms] Warn if `True`/`False`/`Nil` are used as aliases and there is no such alias diff --git a/VERSION b/VERSION index 1f0d2f33519..b6ea925546e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.0-dev +1.16.0-rc.0 diff --git a/bin/elixir b/bin/elixir index cb0a84650b6..ffa46a15047 100755 --- a/bin/elixir +++ b/bin/elixir @@ -1,7 +1,7 @@ #!/bin/sh set -e -ELIXIR_VERSION=1.16.0-dev +ELIXIR_VERSION=1.16.0-rc.0 if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then cat <&2 diff --git a/bin/elixir.bat b/bin/elixir.bat index a222cbcc65e..26594925d79 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -1,6 +1,6 @@ @if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) -set ELIXIR_VERSION=1.16.0-dev +set ELIXIR_VERSION=1.16.0-rc.0 setlocal enabledelayedexpansion if ""%1""=="""" if ""%2""=="""" goto documentation From 2d86cb0027f8361bfad69a231b7a8fa1fa2b0c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 31 Oct 2023 11:59:51 +0100 Subject: [PATCH 011/185] Handle warnings from unquote functions --- lib/elixir/lib/module/parallel_checker.ex | 2 +- .../test/elixir/kernel/diagnostics_test.exs | 2 +- .../elixir/module/types/integration_test.exs | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/module/parallel_checker.ex b/lib/elixir/lib/module/parallel_checker.ex index fa16a310b37..1dab57db3d7 100644 --- a/lib/elixir/lib/module/parallel_checker.ex +++ b/lib/elixir/lib/module/parallel_checker.ex @@ -334,7 +334,7 @@ defmodule Module.ParallelChecker do defp position_to_tuple(position) do case position[:column] do - nil -> position[:line] + nil -> position[:line] || 0 col -> {position[:line], col} end end diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index c2cf1a98aa5..6e30f0f10b4 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -153,7 +153,7 @@ defmodule Kernel.DiagnosticsTest do """ end - test "trim inbetween lines if too many" do + test "trim in between lines if too many" do output = capture_raise( """ diff --git a/lib/elixir/test/elixir/module/types/integration_test.exs b/lib/elixir/test/elixir/module/types/integration_test.exs index d38a6ce6175..ee8cdca18b4 100644 --- a/lib/elixir/test/elixir/module/types/integration_test.exs +++ b/lib/elixir/test/elixir/module/types/integration_test.exs @@ -695,6 +695,29 @@ defmodule Module.Types.IntegrationTest do assert_warnings(files, warnings) end + + test "reports unquote functions" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + def b, do: unquote(&A.a/0) + end + """ + } + + warnings = [ + "A.a/0 is deprecated. oops", + "b.ex: B.b/0" + ] + + assert_warnings(files, warnings) + end end defp assert_warnings(files, expected) when is_binary(expected) do From 43b3e94506aa7fc24bb794830fc76c6bad8ab8ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 31 Oct 2023 20:57:09 +0100 Subject: [PATCH 012/185] Add more examples to unrelated clauses --- .../anti-patterns/design-anti-patterns.md | 71 ++++++++++++------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index f1eb627988a..9fe89c6dea8 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -302,7 +302,7 @@ Using multi-clause functions in Elixir, to group functions of the same name, is #### Example -A frequent example of this usage of multi-clause functions is when developers mix unrelated business logic into the same function definition. Such functions often have generic names or too broad specifications, making it difficult for other developers to understand and maintain them. +A frequent example of this usage of multi-clause functions is when developers mix unrelated business logic into the same function definition, in a way the behaviour of each clause is completely distinct from the other ones. Such functions often have too broad specifications, making it difficult for other developers to understand and maintain them. Some developers may use documentation mechanisms such as `@doc` annotations to compensate for poor code readability, however the documentation itself may end-up full of conditionals to describe how the function behaves for each different argument combination. This is a good indicator that the clauses are ultimately unrelated. @@ -310,65 +310,82 @@ Some developers may use documentation mechanisms such as `@doc` annotations to c @doc """ Updates a struct. -If given a "sharp" product (metal or glass with empty count), -it will... - -If given a blunt product, it will... +If given a product, it will... If given an animal, it will... """ -def update(%Product{count: nil, material: material}) - when material in ["metal", "glass"] do +def update(%Product{count: count, material: material}) do # ... end -def update(%Product{count: count, material: material}) - when count > 0 and material not in ["metal", "glass"] do - # ... -end - -def update(%Animal{count: 1, skin: skin}) - when skin in ["fur", "hairy"] do +def update(%Animal{count: count, skin: skin}) do # ... end ``` +If updating an animal is completely different from updating a product and requires a different set of rules, it may be worth splitting those over different functions or even different modules. + #### Refactoring -As shown below, a possible solution to this anti-pattern is to break the business rules that are mixed up in a single unrelated multi-clause function in several different simple functions. More precise names make the scope of the function clear. Each function can have a specific `@doc`, describing its behavior and parameters received. While this refactoring sounds simple, it can have a lot of impact on the function's current users, so be careful! +As shown below, a possible solution to this anti-pattern is to break the business rules that are mixed up in a single unrelated multi-clause function in simple functions. Each function can have a specific name and `@doc`, describing its behavior and parameters received. While this refactoring sounds simple, it can impact the function's current users, so be careful! ```elixir @doc """ -Updates a "sharp" product. +Updates a product. It will... """ -def update_sharp_product(%Product{count: nil, material: material}) - when material in ["metal", "glass"] do +def update_product(%Product{count: count, material: material}) do # ... end @doc """ -Updates a "blunt" product. +Updates an animal. It will... """ -def update_blunt_product(%Product{count: count, material: material}) - when count > 0 and material not in ["metal", "glass"] do +def update_animal(%Animal{count: count, skin: skin}) do # ... end +``` -@doc """ -Updates an animal. +These functions may still be implemented with multiple clauses, as long as the clauses group related funtionality. For example, `update_product` could be in practice implemented as follows: -It will... -""" -def update_animal(%Animal{count: 1, skin: skin}) - when skin in ["fur", "hairy"] do +```elixir +def update_product(%Product{count: 0}) do + # ... +end + +def update_product(%Product{material: material}) + when material in ["metal", "glass"] do + # ... +end + +def update_product(%Product{material: material}) + when material not in ["metal", "glass"] do # ... end ``` +You can see this pattern in practice within Elixir itself. The `+/2` operator can add `Integer`s and `Float`s together, but not `String`s, which instead use the `<>/2` operator. In this sense, it is reasonable to handle integers and floats in the same operation, but strings are unrelated enough to deserve their own function. + +You will also find examples in Elixir of functions that work with any struct, such as `struct/2`: + +```elixir +iex> struct(URI.parse("/foo/bar"), path: "/bar/baz") +%URI{ + scheme: nil, + userinfo: nil, + host: nil, + port: nil, + path: "/bar/baz", + query: nil, + fragment: nil +} +``` + +The difference here is that the `struct/2` function behaves precisely the same for any struct given, therefore there is no question of how the function handles different inputs. If the behaviour is clear and consistent for all inputs, then the anti-pattern does not take place. + ## Using application configuration for libraries #### Problem From f422b77aaa68106cac324df3329efcee00103eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 1 Nov 2023 10:34:52 +0100 Subject: [PATCH 013/185] Update docs --- lib/elixir/lib/gen_server.ex | 11 ----------- lib/elixir/lib/module.ex | 13 +++++++++++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/lib/elixir/lib/gen_server.ex b/lib/elixir/lib/gen_server.ex index cf581bd5221..02eeb201008 100644 --- a/lib/elixir/lib/gen_server.ex +++ b/lib/elixir/lib/gen_server.ex @@ -8,17 +8,6 @@ defmodule GenServer do will have a standard set of interface functions and include functionality for tracing and error reporting. It will also fit into a supervision tree. - ```mermaid - graph TD - GenServer - GenServer -. reply -.-> A - GenServer -. reply -.-> B - GenServer -. reply -.-> C - A(Client #1) -- request --> GenServer - B(Client #2) -- request --> GenServer - C(Client #3) -- request --> GenServer - ``` - ## Example The GenServer behaviour abstracts the common client-server interaction. diff --git a/lib/elixir/lib/module.ex b/lib/elixir/lib/module.ex index 2184a345694..087c04f1a00 100644 --- a/lib/elixir/lib/module.ex +++ b/lib/elixir/lib/module.ex @@ -326,8 +326,17 @@ defmodule Module do ### `@nifs` (since v1.16.0) A list of functions and their arities which will be overridden - by a native implementation (NIF). See the Erlang documentation - for more information: https://www.erlang.org/doc/man/erl_nif + by a native implementation (NIF). + + defmodule MyLibrary.MyModule do + @nifs [foo: 1, bar: 2] + + def foo(arg1), do: :erlang.nif_error(:not_loaded) + def bar(arg1, arg2), do: :erlang.nif_error(:not_loaded) + end + + See the Erlang documentation for more information: + https://www.erlang.org/doc/man/erl_nif ### `@on_definition` From ae19236e0811c86eee7b38e99e6f6e145659ce68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 1 Nov 2023 11:17:44 +0100 Subject: [PATCH 014/185] Bring behaviour section from website --- lib/elixir/pages/references/typespecs.md | 143 +++++++++++++++++------ 1 file changed, 107 insertions(+), 36 deletions(-) diff --git a/lib/elixir/pages/references/typespecs.md b/lib/elixir/pages/references/typespecs.md index d0281000a13..163219d88d3 100644 --- a/lib/elixir/pages/references/typespecs.md +++ b/lib/elixir/pages/references/typespecs.md @@ -17,7 +17,7 @@ Type specifications (most often referred to as *typespecs*) are defined in diffe * `@callback` * `@macrocallback` -In addition, you can use `@typedoc` to describe a custom `@type` definition. +In addition, you can use `@typedoc` to document a custom `@type` definition. See the "User-defined types" and "Defining a specification" sub-sections below for more information on defining types and typespecs. @@ -250,61 +250,128 @@ Behaviours in Elixir (and Erlang) are a way to separate and abstract the generic A behaviour module defines a set of functions and macros (referred to as *callbacks*) that callback modules implementing that behaviour must export. This "interface" identifies the specific part of the component. For example, the `GenServer` behaviour and functions abstract away all the message-passing (sending and receiving) and error reporting that a "server" process will likely want to implement from the specific parts such as the actions that this server process has to perform. -To define a behaviour module, it's enough to define one or more callbacks in that module. To define callbacks, the `@callback` and `@macrocallback` module attributes can be used (for function callbacks and macro callbacks respectively). +Say we want to implement a bunch of parsers, each parsing structured data: for example, a JSON parser and a MessagePack parser. Each of these two parsers will *behave* the same way: both will provide a `parse/1` function and an `extensions/0` function. The `parse/1` function will return an Elixir representation of the structured data, while the `extensions/0` function will return a list of file extensions that can be used for each type of data (e.g., `.json` for JSON files). - defmodule MyBehaviour do - @callback my_fun(arg :: any) :: any - @macrocallback my_macro(arg :: any) :: Macro.t - end +We can create a `Parser` behaviour: + +```elixir +defmodule Parser do + @doc """ + Parses a string. + """ + @callback parse(String.t) :: {:ok, term} | {:error, atom} + + @doc """ + Lists all supported file extensions. + """ + @callback extensions() :: [String.t] +end +``` As seen in the example above, defining a callback is a matter of defining a specification for that callback, made of: - * the callback name (`my_fun` or `my_macro` in the example) - * the arguments that the callback must accept (`arg :: any` in the example) + * the callback name (`parse` or `extensions` in the example) + * the arguments that the callback must accept (`String.t`) * the *expected* type of the callback return value -### Optional callbacks +Modules adopting the `Parser` behaviour will have to implement all the functions defined with the `@callback` attribute. As you can see, `@callback` expects a function name but also a function specification like the ones used with the `@spec` attribute we saw above. -Optional callbacks are callbacks that callback modules may implement if they want to, but are not required to. Usually, behaviour modules know if they should call those callbacks based on configuration, or they check if the callbacks are defined with `function_exported?/3` or `macro_exported?/3`. +### Implementing behaviours -Optional callbacks can be defined through the `@optional_callbacks` module attribute, which has to be a keyword list with function or macro name as key and arity as value. For example: +Implementing a behaviour is straightforward: - defmodule MyBehaviour do - @callback vital_fun() :: any - @callback non_vital_fun() :: any - @macrocallback non_vital_macro(arg :: any) :: Macro.t - @optional_callbacks non_vital_fun: 0, non_vital_macro: 1 - end +```elixir +defmodule JSONParser do + @behaviour Parser -One example of optional callback in Elixir's standard library is `c:GenServer.format_status/2`. + @impl Parser + def parse(str), do: {:ok, "some json " <> str} # ... parse JSON -### Implementing behaviours + @impl Parser + def extensions, do: [".json"] +end +``` -To specify that a module implements a given behaviour, the `@behaviour` attribute must be used: +```elixir +defmodule CSVParser do + @behaviour Parser - defmodule MyBehaviour do - @callback my_fun(arg :: any) :: any - end + @impl Parser + def parse(str), do: {:ok, "some csv " <> str} # ... parse CSV - defmodule MyCallbackModule do - @behaviour MyBehaviour - def my_fun(arg), do: arg - end + @impl Parser + def extensions, do: [".csv"] +end +``` -If a callback module that implements a given behaviour doesn't export all the functions and macros defined by that behaviour, the user will be notified through warnings during the compilation process (no errors will happen). +If a module adopting a given behaviour doesn't implement one of the callbacks required by that behaviour, a compile-time warning will be generated. -You can also use the `@impl` attribute before a function to denote that particular function is implementation a behaviour: +Furthermore, with `@impl` you can also make sure that you are implementing the **correct** callbacks from the given behaviour in an explicit manner. For example, the following parser implements both `parse` and `extensions`. However, thanks to a typo, `BADParser` is implementing `parse/0` instead of `parse/1`. - defmodule MyCallbackModule do - @behaviour MyBehaviour +```elixir +defmodule BADParser do + @behaviour Parser - @impl true - def my_fun(arg), do: arg - end + @impl Parser + def parse, do: {:ok, "something bad"} + + @impl Parser + def extensions, do: ["bad"] +end +``` + +This code generates a warning letting you know that you are mistakenly implementing `parse/0` instead of `parse/1`. +You can read more about `@impl` in the [module documentation](Module.html#module-impl). + +### Using behaviours + +Behaviours are useful because you can pass modules around as arguments and you can then *call back* to any of the functions specified in the behaviour. For example, we can have a function that receives a filename, several parsers, and parses the file based on its extension: + +```elixir +@spec parse_path(Path.t(), [module()]) :: {:ok, term} | {:error, atom} +def parse_path(filename, parsers) do + with {:ok, ext} <- parse_extension(filename), + {:ok, parser} <- find_parser(ext, parsers), + {:ok, contents} <- File.read(filename) do + parser.parse(contents) + end +end + +defp parse_extension(filename) do + if ext = Path.extname(filename) do + {:ok, ext} + else + {:error, :no_extension} + end +end + +defp find_parser(ext, parsers) do + if parser = Enum.find(parsers, fn parser -> ext in parser.extensions() end) do + {:ok, parser} + else + {:error, :no_matching_parser} + end +end +``` + +You could also invoke any parser directly: `CSVParser.parse(...)`. + +Note you don't need to define a behaviour in order to dynamically dispatch on a module, but those features often go hand in hand. -You can also use `@impl MyBehaviour` to make clearer from which behaviour the callbacks comes from, providing even more context for future readers of your code. +### Optional callbacks + +Optional callbacks are callbacks that callback modules may implement if they want to, but are not required to. Usually, behaviour modules know if they should call those callbacks based on configuration, or they check if the callbacks are defined with `function_exported?/3` or `macro_exported?/3`. -Elixir's standard library contains a few frequently used behaviours such as `GenServer`, `Supervisor`, and `Application`. +Optional callbacks can be defined through the `@optional_callbacks` module attribute, which has to be a keyword list with function or macro name as key and arity as value. For example: + + defmodule MyBehaviour do + @callback vital_fun() :: any + @callback non_vital_fun() :: any + @macrocallback non_vital_macro(arg :: any) :: Macro.t + @optional_callbacks non_vital_fun: 0, non_vital_macro: 1 + end + +One example of optional callback in Elixir's standard library is `c:GenServer.format_status/2`. ### Inspecting behaviours @@ -319,6 +386,10 @@ For example, for the `MyBehaviour` module defined in "Optional callbacks" above: When using `iex`, the `IEx.Helpers.b/1` helper is also available. +## Pitfalls + +There are some known pitfalls when using typespecs, they are documented next. + ## The `string()` type Elixir discourages the use of the `string()` type. The `string()` type refers to Erlang strings, which are known as "charlists" in Elixir. They do not refer to Elixir strings, which are UTF-8 encoded binaries. To avoid confusion, if you attempt to use the type `string()`, Elixir will emit a warning. You should use `charlist()`, `nonempty_charlist()`, `binary()` or `String.t()` accordingly, or any of the several literal representations for these types. From 0f65cb065a648753dff93c9ae3716d6579b9abb3 Mon Sep 17 00:00:00 2001 From: Panagiotis Nezis Date: Wed, 1 Nov 2023 13:27:21 +0200 Subject: [PATCH 015/185] Additional remarks for application config anti-pattern for Mix tasks (#13057) --- .../anti-patterns/design-anti-patterns.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index 9fe89c6dea8..4f94b6d159b 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -444,3 +444,35 @@ iex> DashSplitter.split("Lucas-Francisco-da-Matta-Vegi", [parts: 5]) iex> DashSplitter.split("Lucas-Francisco-da-Matta-Vegi") #<= default config is used! ["Lucas", "Francisco-da-Matta-Vegi"] ``` + +#### Additional Remarks + +For Mix tasks and related tools, it may be necessary to provide per-project configuration. For example, imagine you have a `:linter` project, which supports setting the output file and the verbosity level. You may choose to configure it through application environment: + +```elixir +config :linter, + output_file: "/path/to/output.json", + verbosity: 3 +``` + +However, `Mix` allows tasks to read per-project configuration via `Mix.Project.config/0`. In this case, you can configure the `:linter` directly in the `mix.exs` file: + +```elixir +def project do + [ + app: :my_app, + version: "1.0.0", + linter: [ + output_file: "/path/to/output.json", + verbosity: 3 + ], + ... + ] +end +``` + +Additonally, if a Mix task is available, you can also accept these options as command line arguments (see `OptionParser`): + +```bash +mix linter --output-file /path/to/output.json --verbosity 3 +``` From 3822609621e0a7e20f77b65185c806dcd19e1873 Mon Sep 17 00:00:00 2001 From: rktjmp Date: Thu, 2 Nov 2023 01:33:01 +1100 Subject: [PATCH 016/185] Restore GenServer introduction mermaid graph (#13058) Restores graph removed in f5a61d1, with correct request -> reply arrow ordering. --- lib/elixir/lib/gen_server.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/elixir/lib/gen_server.ex b/lib/elixir/lib/gen_server.ex index 02eeb201008..a2884ebbdbb 100644 --- a/lib/elixir/lib/gen_server.ex +++ b/lib/elixir/lib/gen_server.ex @@ -8,6 +8,13 @@ defmodule GenServer do will have a standard set of interface functions and include functionality for tracing and error reporting. It will also fit into a supervision tree. + ```mermaid + graph BT + C(Client #3) ~~~ B(Client #2) ~~~ A(Client #1) + A & B & C -->|request| GenServer + GenServer -.->|reply| A & B & C + ``` + ## Example The GenServer behaviour abstracts the common client-server interaction. From e6b641a0757a6c909c4745b97dbc2e3c07a04263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 1 Nov 2023 16:11:09 +0100 Subject: [PATCH 017/185] Fix typo on docs --- lib/elixir/lib/inspect/algebra.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/elixir/lib/inspect/algebra.ex b/lib/elixir/lib/inspect/algebra.ex index cfd790f77f0..5e213b37798 100644 --- a/lib/elixir/lib/inspect/algebra.ex +++ b/lib/elixir/lib/inspect/algebra.ex @@ -143,9 +143,8 @@ defmodule Inspect.Opts do function as this must be controlled by applications. Libraries should instead define their own structs with custom inspect implementations. If a library must change the default inspect - function, then it is best to define to ask users of your library - to explicitly call `default_inspect_fun/1` with your function of - choice. + function, then it is best to ask users of your library to explicitly + call `default_inspect_fun/1` with your function of choice. The default is `Inspect.inspect/2`. From f5e543576e556e40a3f4fc6b96f2fa78b0ae7342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 1 Nov 2023 16:33:17 +0100 Subject: [PATCH 018/185] Use explicit/implicit vs manual/automatic --- lib/elixir/lib/supervisor.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/supervisor.ex b/lib/elixir/lib/supervisor.ex index e56527fa994..4ddb7f2bbaf 100644 --- a/lib/elixir/lib/supervisor.ex +++ b/lib/elixir/lib/supervisor.ex @@ -413,10 +413,10 @@ defmodule Supervisor do The difference between the two approaches is that a module-based supervisor gives you more direct control over how the supervisor is initialized. Instead of calling `Supervisor.start_link/2` with - a list of child specifications that are automatically initialized, we manually - initialize the children by calling `Supervisor.init/2` inside its - `c:init/1` callback. `Supervisor.init/2` accepts the same `:strategy`, - `:max_restarts`, and `:max_seconds` options as `start_link/2`. + a list of child specifications that are implicitly initialized for us, + we must explicitly initialize the children by calling `Supervisor.init/2` + inside its `c:init/1` callback. `Supervisor.init/2` accepts the same + `:strategy`, `:max_restarts`, and `:max_seconds` options as `start_link/2`. > #### `use Supervisor` {: .info} > From 741757f2a7e5a6d932e21e9bb734efa0ed5a31c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Samson?= Date: Thu, 2 Nov 2023 08:11:31 +0100 Subject: [PATCH 019/185] Lazily evaluate File.cwd! in Path.expand and Path.absname (#13061) do not crash with File.Error with already absolute paths if File.cwd returns error or nil --- lib/elixir/lib/path.ex | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/path.ex b/lib/elixir/lib/path.ex index 472ea988f48..0955e72c505 100644 --- a/lib/elixir/lib/path.ex +++ b/lib/elixir/lib/path.ex @@ -44,7 +44,7 @@ defmodule Path do """ @spec absname(t) :: binary def absname(path) do - absname(path, File.cwd!()) + absname(path, &File.cwd!/0) end @doc """ @@ -71,13 +71,27 @@ defmodule Path do case type(path) do :relative -> + relative_to = + if is_function(relative_to, 0) do + relative_to.() + else + relative_to + end + absname_join([relative_to, path]) :absolute -> absname_join([path]) :volumerelative -> - relative_to = IO.chardata_to_string(relative_to) + relative_to = + if is_function(relative_to, 0) do + relative_to.() + else + relative_to + end + |> IO.chardata_to_string() + absname_vr(split(path), split(relative_to), relative_to) end end @@ -163,7 +177,7 @@ defmodule Path do """ @spec expand(t) :: binary def expand(path) do - expand_dot(absname(expand_home(path), File.cwd!())) + expand_dot(absname(expand_home(path), &File.cwd!/0)) end @doc """ @@ -192,7 +206,7 @@ defmodule Path do """ @spec expand(t, t) :: binary def expand(path, relative_to) do - expand_dot(absname(absname(expand_home(path), expand_home(relative_to)), File.cwd!())) + expand_dot(absname(absname(expand_home(path), expand_home(relative_to)), &File.cwd!/0)) end @doc """ From aabe5dad76d007a3444af23eba2359820c289d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 2 Nov 2023 08:29:12 +0100 Subject: [PATCH 020/185] Do not use Erlang/OTP 26.1 on CI (#13062) It has a bug when looking up mismatched module names. --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe1f6b4b55..4a35c6ad3f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,10 +3,10 @@ name: CI on: push: paths-ignore: - - 'lib/**/*.md' + - "lib/**/*.md" pull_request: paths-ignore: - - 'lib/**/*.md' + - "lib/**/*.md" env: ELIXIR_ASSERT_TIMEOUT: 2000 @@ -24,12 +24,12 @@ jobs: fail-fast: false matrix: include: - - otp_version: '26.0' + - otp_version: "26.0" otp_latest: true - - otp_version: '25.3' - - otp_version: '25.0' - - otp_version: '24.3' - - otp_version: '24.0' + - otp_version: "25.3" + - otp_version: "25.0" + - otp_version: "24.3" + - otp_version: "24.0" - otp_version: master development: true - otp_version: maint @@ -77,7 +77,7 @@ jobs: name: Windows Server 2019, Erlang/OTP ${{ matrix.otp_version }} strategy: matrix: - otp_version: ['24', '25', '26'] + otp_version: ["24", "25", "26.0"] runs-on: windows-2019 steps: - name: Configure Git From 0220d81513c79d495ab9be8fe2a60ae0ae563dea Mon Sep 17 00:00:00 2001 From: Panagiotis Nezis Date: Thu, 2 Nov 2023 09:57:35 +0200 Subject: [PATCH 021/185] Support --sparse in archive.install and escript.install (#13059) --- lib/mix/lib/mix/local/installer.ex | 24 +++++++++++--- lib/mix/lib/mix/tasks/archive.install.ex | 4 +++ lib/mix/lib/mix/tasks/escript.install.ex | 4 +++ lib/mix/test/mix/local/installer_test.exs | 40 +++++++++++++++++++---- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/lib/mix/lib/mix/local/installer.ex b/lib/mix/lib/mix/local/installer.ex index ce23c02b946..ea39b6d78fd 100644 --- a/lib/mix/lib/mix/local/installer.ex +++ b/lib/mix/lib/mix/local/installer.ex @@ -245,7 +245,7 @@ defmodule Mix.Local.Installer do end defp git_fetcher(url, git_config, opts) do - git_opts = git_config ++ [git: url, submodules: opts[:submodules]] + git_opts = git_config ++ [git: url, submodules: opts[:submodules], sparse: opts[:sparse]] app_name = if opts[:app] do @@ -347,9 +347,7 @@ defmodule Mix.Local.Installer do File.cwd!() end) - package_name = elem(dep_spec, 0) - package_name_string = Atom.to_string(package_name) - package_path = Path.join([tmp_path, "deps", package_name_string]) + {package_name, package_path} = package_name_path(dep_spec, tmp_path) post_config = [ deps_path: Path.join(tmp_path, "deps"), @@ -368,6 +366,24 @@ defmodule Mix.Local.Installer do :code.delete(Mix.Local.Installer.Fetcher) end + defp package_name_path(dep_spec, tmp_path) do + package_name = elem(dep_spec, 0) + package_name_string = Atom.to_string(package_name) + package_path = Path.join([tmp_path, "deps", package_name_string, maybe_sparse_dir(dep_spec)]) + + {package_name, package_path} + end + + defp maybe_sparse_dir({_app, opts}) when is_list(opts) do + if opts[:git] do + opts[:sparse] || "" + else + "" + end + end + + defp maybe_sparse_dir(_dep_spec), do: "" + defp in_fetcher(_mix_exs) do Mix.Task.run("deps.get", ["--only", Atom.to_string(Mix.env())]) end diff --git a/lib/mix/lib/mix/tasks/archive.install.ex b/lib/mix/lib/mix/tasks/archive.install.ex index 19c47dfb172..39abd5dd514 100644 --- a/lib/mix/lib/mix/tasks/archive.install.ex +++ b/lib/mix/lib/mix/tasks/archive.install.ex @@ -47,6 +47,9 @@ defmodule Mix.Tasks.Archive.Install do * `--submodules` - fetches repository submodules before building archive from Git or GitHub + * `--sparse` - checkout a single directory inside the Git repository and use + it as the archive root directory + * `--app` - specifies a custom app name to be used for building the archive from Git, GitHub, or Hex @@ -63,6 +66,7 @@ defmodule Mix.Tasks.Archive.Install do force: :boolean, sha512: :string, submodules: :boolean, + sparse: :string, app: :string, organization: :string, repo: :string, diff --git a/lib/mix/lib/mix/tasks/escript.install.ex b/lib/mix/lib/mix/tasks/escript.install.ex index d06995a951b..ba1b9a46ed1 100644 --- a/lib/mix/lib/mix/tasks/escript.install.ex +++ b/lib/mix/lib/mix/tasks/escript.install.ex @@ -47,6 +47,9 @@ defmodule Mix.Tasks.Escript.Install do * `--submodules` - fetches repository submodules before building escript from Git or GitHub + * `--sparse` - checkout a single directory inside the Git repository and use + it as the escript project directory + * `--app` - specifies a custom app name to be used for building the escript from Git, GitHub, or Hex @@ -66,6 +69,7 @@ defmodule Mix.Tasks.Escript.Install do force: :boolean, sha512: :string, submodules: :boolean, + sparse: :string, app: :string, organization: :string, repo: :string, diff --git a/lib/mix/test/mix/local/installer_test.exs b/lib/mix/test/mix/local/installer_test.exs index 5c5fdcae0c2..bf10c4d1373 100644 --- a/lib/mix/test/mix/local/installer_test.exs +++ b/lib/mix/test/mix/local/installer_test.exs @@ -30,50 +30,76 @@ defmodule Mix.Local.InstallerTest do test "parse_args Git" do args = ["git", "https://example.com/user/repo.git"] - opts = [git: "https://example.com/user/repo.git", submodules: nil] + opts = [git: "https://example.com/user/repo.git", submodules: nil, sparse: nil] assert Mix.Local.Installer.parse_args(args, []) == {:fetcher, {:"new package", opts}} end test "parse_args Git branch" do args = ["git", "https://example.com/user/repo.git", "branch", "not_main"] - opts = [branch: "not_main", git: "https://example.com/user/repo.git", submodules: nil] + + opts = [ + branch: "not_main", + git: "https://example.com/user/repo.git", + submodules: nil, + sparse: nil + ] assert Mix.Local.Installer.parse_args(args, []) == {:fetcher, {:"new package", opts}} end test "parse_args Git ref" do args = ["git", "https://example.com/user/repo.git", "ref", "not_main"] - opts = [ref: "not_main", git: "https://example.com/user/repo.git", submodules: nil] + + opts = [ + ref: "not_main", + git: "https://example.com/user/repo.git", + submodules: nil, + sparse: nil + ] assert Mix.Local.Installer.parse_args(args, []) == {:fetcher, {:"new package", opts}} end test "parse_args Git tag" do args = ["git", "https://example.com/user/repo.git", "tag", "not_main"] - opts = [tag: "not_main", git: "https://example.com/user/repo.git", submodules: nil] + + opts = [ + tag: "not_main", + git: "https://example.com/user/repo.git", + submodules: nil, + sparse: nil + ] assert Mix.Local.Installer.parse_args(args, []) == {:fetcher, {:"new package", opts}} end test "parse_args Git submodules" do args = ["git", "https://example.com/user/repo.git"] - opts = [git: "https://example.com/user/repo.git", submodules: true] + opts = [git: "https://example.com/user/repo.git", submodules: true, sparse: nil] assert Mix.Local.Installer.parse_args(args, submodules: true) == {:fetcher, {:"new package", opts}} end + test "parse_args Git sparse" do + args = ["git", "https://example.com/user/repo.git"] + opts = [git: "https://example.com/user/repo.git", submodules: nil, sparse: "foo"] + + assert Mix.Local.Installer.parse_args(args, sparse: "foo") == + {:fetcher, {:"new package", opts}} + end + test "parse_args Git app" do args = ["git", "https://example.com/user/repo.git"] - opts = [git: "https://example.com/user/repo.git", submodules: nil] + opts = [git: "https://example.com/user/repo.git", submodules: nil, sparse: nil] assert Mix.Local.Installer.parse_args(args, app: "my_app") == {:fetcher, {:my_app, opts}} end test "parse_args GitHub" do args = ["github", "user/repo"] - opts = [git: "https://github.com/user/repo.git", submodules: nil] + opts = [git: "https://github.com/user/repo.git", submodules: nil, sparse: nil] assert Mix.Local.Installer.parse_args(args, []) == {:fetcher, {:"new package", opts}} end From 23c27b629d6bbc0d60699eb064af9582d6934b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 2 Nov 2023 10:19:11 +0100 Subject: [PATCH 022/185] Warn if both :applications and :extra_applications are used --- lib/mix/lib/mix/tasks/compile.app.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/mix/lib/mix/tasks/compile.app.ex b/lib/mix/lib/mix/tasks/compile.app.ex index 4aaaecf6cc9..f54b3a6f338 100644 --- a/lib/mix/lib/mix/tasks/compile.app.ex +++ b/lib/mix/lib/mix/tasks/compile.app.ex @@ -344,6 +344,13 @@ defmodule Mix.Tasks.Compile.App do defp handle_extra_applications(properties, config) do {extra, properties} = Keyword.pop(properties, :extra_applications, []) + if extra != [] and Keyword.has_key?(properties, :applications) do + Mix.shell().error( + "both :extra_applications and :applications was found in your mix.exs. " <> + "You most likely want to remove the :applications key, as all applications are derived from your dependencies" + ) + end + {all, optional} = project_apps(properties, config, extra, fn -> apps_from_runtime_prod_deps(properties, config) From 8d545610d28f22d7e35c6f4c0155d7b08942ab8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=81=C4=99picki?= Date: Thu, 2 Nov 2023 13:08:28 +0100 Subject: [PATCH 023/185] Fix Path.absname/2 spec (#13065) --- lib/elixir/lib/path.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/path.ex b/lib/elixir/lib/path.ex index 0955e72c505..0f18487a101 100644 --- a/lib/elixir/lib/path.ex +++ b/lib/elixir/lib/path.ex @@ -51,10 +51,10 @@ defmodule Path do Builds a path from `relative_to` to `path`. If `path` is already an absolute path, `relative_to` is ignored. See also - `relative_to/3`. + `relative_to/3`. `relative_to` is either a path or an anonymous function, + which is invoked only when necessary, that returns a path. - Unlike `expand/2`, no attempt is made to - resolve `..`, `.` or `~`. + Unlike `expand/2`, no attempt is made to resolve `..`, `.` or `~`. ## Examples @@ -65,7 +65,7 @@ defmodule Path do "bar/../x" """ - @spec absname(t, t) :: binary + @spec absname(t, t | (-> t)) :: binary def absname(path, relative_to) do path = IO.chardata_to_string(path) From a7216decad7e73dc610237ea587fe8f9f2ff6c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 2 Nov 2023 17:57:49 +0100 Subject: [PATCH 024/185] Add offset option to File.stream! (#13063) --- lib/elixir/lib/file.ex | 3 ++ lib/elixir/lib/file/stream.ex | 46 +++++++++++++++++++-- lib/elixir/test/elixir/file/stream_test.exs | 30 ++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/file.ex b/lib/elixir/lib/file.ex index f9f928146f9..b65c51478c6 100644 --- a/lib/elixir/lib/file.ex +++ b/lib/elixir/lib/file.ex @@ -133,6 +133,7 @@ defmodule File do @type stream_mode :: encoding_mode() + | read_offset_mode() | :append | :compressed | :delayed_write @@ -140,6 +141,8 @@ defmodule File do | {:read_ahead, pos_integer | false} | {:delayed_write, non_neg_integer, non_neg_integer} + @type read_offset_mode :: {:read_offset, non_neg_integer()} + @type erlang_time :: {{year :: non_neg_integer(), month :: 1..12, day :: 1..31}, {hour :: 0..23, minute :: 0..59, second :: 0..59}} diff --git a/lib/elixir/lib/file/stream.ex b/lib/elixir/lib/file/stream.ex index cd23f2b0b54..4633ca1f1c2 100644 --- a/lib/elixir/lib/file/stream.ex +++ b/lib/elixir/lib/file/stream.ex @@ -18,6 +18,12 @@ defmodule File.Stream do @doc false def __build__(path, line_or_bytes, modes) do + with {:read_offset, offset} <- :lists.keyfind(:read_offset, 1, modes), + false <- is_integer(offset) and offset >= 0 do + raise ArgumentError, + "expected :read_offset to be a non-negative integer, got: #{inspect(offset)}" + end + raw = :lists.keyfind(:encoding, 1, modes) == false modes = @@ -88,7 +94,7 @@ defmodule File.Stream do start_fun = fn -> case File.Stream.__open__(stream, read_modes(modes)) do {:ok, device} -> - if :trim_bom in modes, do: trim_bom(device, raw) |> elem(0), else: device + skip_bom_and_offset(device, raw, modes) {:error, reason} -> raise File.Error, reason: reason, action: "stream", path: stream.path @@ -104,9 +110,14 @@ defmodule File.Stream do Stream.resource(start_fun, next_fun, &:file.close/1).(acc, fun) end - def count(%{modes: modes, line_or_bytes: :line, path: path} = stream) do + def count(%{modes: modes, line_or_bytes: :line, path: path, raw: raw} = stream) do pattern = :binary.compile_pattern("\n") - counter = &count_lines(&1, path, pattern, read_function(stream), 0) + + counter = fn device -> + device = skip_bom_and_offset(device, raw, modes) + count_lines(device, path, pattern, read_function(stream), 0) + end + {:ok, open!(stream, modes, counter)} end @@ -116,8 +127,11 @@ defmodule File.Stream do {:error, __MODULE__} {:ok, %{size: size}} -> + bom_offset = count_raw_bom(stream, modes) + offset = get_read_offset(modes) + size = max(size - bom_offset - offset, 0) remainder = if rem(size, bytes) == 0, do: 0, else: 1 - {:ok, div(size, bytes) + remainder - count_raw_bom(stream, modes)} + {:ok, div(size, bytes) + remainder} {:error, reason} -> raise File.Error, reason: reason, action: "stream", path: path @@ -158,6 +172,23 @@ defmodule File.Stream do end end + defp skip_bom_and_offset(device, raw, modes) do + device = + if :trim_bom in modes do + device |> trim_bom(raw) |> elem(0) + else + device + end + + offset = get_read_offset(modes) + + if offset > 0 do + {:ok, _} = :file.position(device, {:cur, offset}) + end + + device + end + defp trim_bom(device, true) do bom_length = device |> IO.binread(4) |> bom_length() {:ok, new_pos} = :file.position(device, bom_length) @@ -183,6 +214,13 @@ defmodule File.Stream do defp bom_length(<<254, 255, 0, 0, _rest::binary>>), do: 4 defp bom_length(_binary), do: 0 + def get_read_offset(modes) do + case :lists.keyfind(:read_offset, 1, modes) do + {:read_offset, offset} -> offset + false -> 0 + end + end + defp read_modes(modes) do for mode <- modes, mode not in [:write, :append, :trim_bom], do: mode end diff --git a/lib/elixir/test/elixir/file/stream_test.exs b/lib/elixir/test/elixir/file/stream_test.exs index 2aa77030661..e61f7b5d431 100644 --- a/lib/elixir/test/elixir/file/stream_test.exs +++ b/lib/elixir/test/elixir/file/stream_test.exs @@ -141,6 +141,35 @@ defmodule File.StreamTest do end end + test "supports byte offset" do + src = fixture_path("file.txt") + + assert @node + |> stream!(src, [{:read_offset, 0}]) + |> Enum.take(1) == ["FOO\n"] + + assert @node + |> stream!(src, [{:read_offset, 1}]) + |> Enum.take(1) == ["OO\n"] + + assert @node + |> stream!(src, [{:read_offset, 4}]) + |> Enum.take(1) == [] + + assert @node |> stream!(src, 1, [{:read_offset, 1}]) |> Enum.count() == 3 + assert @node |> stream!(src, 1, [{:read_offset, 4}]) |> Enum.count() == 0 + end + + test "applies offset after trimming BOM" do + src = fixture_path("utf8_bom.txt") + + assert @node + |> stream!(src, [:trim_bom, {:read_offset, 4}]) + |> Enum.take(1) == ["сский\n"] + + assert @node |> stream!(src, 1, [:trim_bom, {:read_offset, 4}]) |> Enum.count() == 15 + end + test "keeps BOM when raw" do src = fixture_path("utf8_bom.txt") @@ -169,6 +198,7 @@ defmodule File.StreamTest do assert @node |> stream!(src, [:trim_bom]) |> Enum.count() == 2 assert @node |> stream!(src, 1, [:trim_bom]) |> Enum.count() == 19 + assert @node |> stream!(src, 2, [:trim_bom]) |> Enum.count() == 10 end test "keeps BOM with utf8 encoding" do From d5316d558f13a936139928ef866fab52b84f664b Mon Sep 17 00:00:00 2001 From: Cameron Duley Date: Thu, 2 Nov 2023 14:29:11 -0400 Subject: [PATCH 025/185] Add `String.replace_invalid/2` (#13067) --- lib/elixir/lib/string.ex | 105 +++++++++++++++++++++++++ lib/elixir/test/elixir/string_test.exs | 27 +++++++ 2 files changed, 132 insertions(+) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index 29f01669165..562e7b8ca23 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -1871,6 +1871,111 @@ defmodule String do end end + @doc ~S""" + Returns a new string created by replacing all invalid bytes with `replacement` (`"�"` by default). + + ## Examples + + iex> String.replace_invalid("asd" <> <<0xFF::8>>) + "asd�" + + iex> String.replace_invalid("nem rán bề bề") + "nem rán bề bề" + + iex> String.replace_invalid("nem rán b" <> <<225, 187>> <> " bề") + "nem rán b� bề" + + iex> String.replace_invalid("nem rán b" <> <<225, 187>> <> " bề", "ERROR!") + "nem rán bERROR! bề" + """ + def replace_invalid(string, replacement \\ "�") + when is_binary(string) and is_binary(replacement) do + do_replace_invalid(string, replacement, <<>>) + end + + # Valid ASCII (for better average speed) + defp do_replace_invalid(<>, rep, acc) + when ascii in 0..127 and n_lead != 0b10 do + do_replace_invalid(<>, rep, <>) + end + + # Valid UTF-8 + defp do_replace_invalid(<>, rep, acc) do + do_replace_invalid(rest, rep, <>) + end + + # 2/3 truncated sequence + defp do_replace_invalid(<<0b1110::4, i::4, 0b10::2, ii::6>>, rep, acc) do + <> = <> + <> + end + + defp do_replace_invalid(<<0b1110::4, i::4, 0b10::2, ii::6, n_lead::2, rest::bits>>, rep, acc) + when n_lead != 0b10 do + <> = <> + + do_replace_invalid( + <>, + rep, + <> + ) + end + + # 2/4 + defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6>>, rep, acc) do + <> = <> + <> + end + + defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6, n_lead::2, rest::bits>>, rep, acc) + when n_lead != 0b10 do + <> = <> + + do_replace_invalid( + <>, + rep, + <> + ) + end + + # 3/4 + defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6, 0b10::2, iii::6>>, rep, acc) do + <> = <> + <> + end + + defp do_replace_invalid( + <<0b11110::5, i::3, 0b10::2, ii::6, 0b10::2, iii::6, n_lead::2, rest::bits>>, + rep, + acc + ) + when n_lead != 0b10 do + <> = <> + + do_replace_invalid( + <>, + rep, + <> + ) + end + + # any other invalid bytes + defp do_replace_invalid(<<_, rest::bits>>, rep, acc), + do: do_replace_invalid(rest, rep, <>) + + defp do_replace_invalid(<<>>, _, acc), do: acc + + # bounds-checking truncated code points for overlong encodings + defp replace_invalid_ii_of_iii(tcp, rep) when tcp >= 32 and tcp <= 863, do: rep + defp replace_invalid_ii_of_iii(tcp, rep) when tcp >= 896 and tcp <= 1023, do: rep + defp replace_invalid_ii_of_iii(_, rep), do: rep <> rep + + defp replace_invalid_ii_of_iiii(tcp, rep) when tcp >= 16 and tcp <= 271, do: rep + defp replace_invalid_ii_of_iiii(_, rep), do: rep <> rep + + defp replace_invalid_iii_of_iiii(tcp, rep) when tcp >= 1024 and tcp <= 17407, do: rep + defp replace_invalid_iii_of_iiii(_, rep), do: rep <> rep <> rep + @doc ~S""" Splits the string into chunks of characters that share a common trait. diff --git a/lib/elixir/test/elixir/string_test.exs b/lib/elixir/test/elixir/string_test.exs index d644796b3a4..8a411eb78d1 100644 --- a/lib/elixir/test/elixir/string_test.exs +++ b/lib/elixir/test/elixir/string_test.exs @@ -826,6 +826,33 @@ defmodule StringTest do refute String.valid?("asdasdasd" <> <<0xFFFF::16>>, :fast_ascii) end + test "replace_invalid" do + assert String.replace_invalid("") === "" + assert String.replace_invalid(<<0xFF>>) === "�" + assert String.replace_invalid(<<0xFF, 0xFF, 0xFF>>) === "���" + + # Valid ASCII + assert String.replace_invalid("hello") === "hello" + + # Valid UTF-8 + assert String.replace_invalid("こんにちは") === "こんにちは" + + # 2/3 byte truncated "ề" + assert String.replace_invalid(<<225, 187>>) === "�" + assert String.replace_invalid("nem rán b" <> <<225, 187>> <> " bề") === "nem rán b� bề" + + # 2/4 byte truncated "😔" + assert String.replace_invalid(<<240, 159>>) === "�" + assert String.replace_invalid("It's so over " <> <<240, 159>>) === "It's so over �" + + # 3/4 byte truncated "😃" + assert String.replace_invalid(<<240, 159, 152>>) === "�" + assert String.replace_invalid("We're so back " <> <<240, 159, 152>>) === "We're so back �" + + # 3 byte overlong "e" + assert String.replace_invalid(<<0b11100000, 0b10000001, 0b10100101>>) === "���" + end + test "chunk/2 with :valid trait" do assert String.chunk("", :valid) == [] From 3a516a3f2b31e01d352e98d1177c532dd71bddcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 2 Nov 2023 19:31:58 +0100 Subject: [PATCH 026/185] Docs to new options and functions --- lib/elixir/lib/file.ex | 6 ++++-- lib/elixir/lib/string.ex | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/file.ex b/lib/elixir/lib/file.ex index b65c51478c6..d302d971418 100644 --- a/lib/elixir/lib/file.ex +++ b/lib/elixir/lib/file.ex @@ -1751,13 +1751,15 @@ defmodule File do One may also consider passing the `:delayed_write` option if the stream is meant to be written to under a tight loop. - ## Byte order marks + ## Byte order marks an dread offset If you pass `:trim_bom` in the modes parameter, the stream will trim UTF-8, UTF-16 and UTF-32 byte order marks when reading from file. Note that this function does not try to discover the file encoding - based on BOM. + based on BOM. From Elixir v1.16.0, you may also pass a `:read_offset` + that is skipped whenever enumerating the stream (if both `:read_offset` + and `:trim_bom` are given, the offset is skipped after the BOM). ## Examples diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index 562e7b8ca23..5bfadfd5627 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -1888,6 +1888,7 @@ defmodule String do iex> String.replace_invalid("nem rán b" <> <<225, 187>> <> " bề", "ERROR!") "nem rán bERROR! bề" """ + @doc since: "1.16.0" def replace_invalid(string, replacement \\ "�") when is_binary(string) and is_binary(replacement) do do_replace_invalid(string, replacement, <<>>) From 178b654a1be91971f9f8e8a76008347daee2dbef Mon Sep 17 00:00:00 2001 From: Rich Morin Date: Thu, 2 Nov 2023 13:04:21 -0700 Subject: [PATCH 027/185] Fix typo (#13068) --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index 22479a3a80b..2d0147f5a6a 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -220,7 +220,7 @@ defmodule MyRequestHandler do end ``` -All valid statuses all defined as atoms within the same module, and that's enough. If you want to be explicit, you could also have a function that lists them: +All valid statuses are defined as atoms within the same module, and that's enough. If you want to be explicit, you could also have a function that lists them: ```elixir def valid_statuses do From d28084684337b277467ccdc19844541e5f04f461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=81=C4=99picki?= Date: Fri, 3 Nov 2023 08:08:07 +0100 Subject: [PATCH 028/185] Fix typo: an dread -> and read (#13069) --- lib/elixir/lib/file.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/lib/file.ex b/lib/elixir/lib/file.ex index d302d971418..ecf4b8550dd 100644 --- a/lib/elixir/lib/file.ex +++ b/lib/elixir/lib/file.ex @@ -1751,7 +1751,7 @@ defmodule File do One may also consider passing the `:delayed_write` option if the stream is meant to be written to under a tight loop. - ## Byte order marks an dread offset + ## Byte order marks and read offset If you pass `:trim_bom` in the modes parameter, the stream will trim UTF-8, UTF-16 and UTF-32 byte order marks when reading from file. From 7923e4f383f4672aea43a1e424330d8fad82d11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Samson?= Date: Fri, 3 Nov 2023 11:53:30 +0100 Subject: [PATCH 029/185] Elixir 1.14.5 supports Erlang/OTP 26 (#13071) --- lib/elixir/pages/references/compatibility-and-deprecations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/references/compatibility-and-deprecations.md b/lib/elixir/pages/references/compatibility-and-deprecations.md index 48441356ce7..2434848ad8e 100644 --- a/lib/elixir/pages/references/compatibility-and-deprecations.md +++ b/lib/elixir/pages/references/compatibility-and-deprecations.md @@ -44,7 +44,7 @@ Elixir version | Supported Erlang/OTP versions :------------- | :------------------------------- 1.16 | 24 - 26 1.15 | 24 - 26 -1.14 | 23 - 25 +1.14 | 23 - 25 (and Erlang/OTP 26 from v1.14.5) 1.13 | 22 - 24 (and Erlang/OTP 25 from v1.13.4) 1.12 | 22 - 24 1.11 | 21 - 23 (and Erlang/OTP 24 from v1.11.4) From 6745f775b2c4e7ddcd748132c903a300ef7f0f67 Mon Sep 17 00:00:00 2001 From: Ioannis Kyriazis Date: Fri, 3 Nov 2023 18:23:45 -0400 Subject: [PATCH 030/185] is -> us (#13072) --- lib/elixir/pages/getting-started/io-and-the-file-system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/getting-started/io-and-the-file-system.md b/lib/elixir/pages/getting-started/io-and-the-file-system.md index a3168b2536d..57a4760633e 100644 --- a/lib/elixir/pages/getting-started/io-and-the-file-system.md +++ b/lib/elixir/pages/getting-started/io-and-the-file-system.md @@ -129,7 +129,7 @@ iex> IO.write(pid, "hello") After `IO.write/2`, we can see the request sent by the `IO` module printed out (a four-elements tuple). Soon after that, we see that it fails since the `IO` module expected some kind of result, which we did not supply. -By modeling IO devices with processes, the Erlang VM allows is to even read and write to files across nodes. Neat! +By modeling IO devices with processes, the Erlang VM allows us to even read and write to files across nodes. Neat! ## `iodata` and `chardata` From ec8782a486191d556805554aa459ace84499a47d Mon Sep 17 00:00:00 2001 From: Tony Dang Date: Sat, 4 Nov 2023 00:59:00 -0700 Subject: [PATCH 031/185] Fix typo in "Getting Started - Enumerables and Streams" docs (#13073) --- lib/elixir/pages/getting-started/enumerable-and-streams.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/getting-started/enumerable-and-streams.md b/lib/elixir/pages/getting-started/enumerable-and-streams.md index 7dc7d98e987..bc4f81516f0 100644 --- a/lib/elixir/pages/getting-started/enumerable-and-streams.md +++ b/lib/elixir/pages/getting-started/enumerable-and-streams.md @@ -119,6 +119,6 @@ iex> Enum.take(stream, 10) The example above will fetch the first 10 lines of the file you have selected. This means streams can be very useful for handling large files or even slow resources like network resources. -The `Enum` and `Stream` modules provide a wide-range functions but know all of them at heart. Familiarize yourself with `Enum.map/2`, `Enum.reduce/3` and other functions with either `map` or `reduce` in their names, and you will naturally build an intuition around the most important use cases. You may also focus on the `Enum` module first and only move to `Stream` for the particular scenarios where laziness is required, to either deal with slow resources or large, possibly infinite, collections. +The `Enum` and `Stream` modules provide a wide range of functions, but you don't have to know all of them by heart. Familiarize yourself with `Enum.map/2`, `Enum.reduce/3` and other functions with either `map` or `reduce` in their names, and you will naturally build an intuition around the most important use cases. You may also focus on the `Enum` module first and only move to `Stream` for the particular scenarios where laziness is required, to either deal with slow resources or large, possibly infinite, collections. Next, we'll look at a feature central to Elixir, Processes, which allows us to write concurrent, parallel and distributed programs in an easy and understandable way. From daab3f80a07c9df6a8d6451687b8ba76e5999e44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Samson?= Date: Sun, 5 Nov 2023 20:15:15 +0100 Subject: [PATCH 032/185] Fix crashes when :beam_lib.info(beam) returns error (#13075) --- lib/elixir/lib/code.ex | 3 ++- lib/elixir/lib/code/typespec.ex | 3 ++- lib/iex/lib/iex/helpers.ex | 3 ++- lib/iex/lib/iex/info.ex | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 09099667a5c..a700b71b674 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -2057,7 +2057,8 @@ defmodule Code do defp get_beam_and_path(module) do with {^module, beam, filename} <- :code.get_object_code(module), - {:ok, ^module} <- beam |> :beam_lib.info() |> Keyword.fetch(:module) do + info_pairs when is_list(info_pairs) <- :beam_lib.info(beam), + {:ok, ^module} <- Keyword.fetch(info_pairs, :module) do {beam, filename} else _ -> :error diff --git a/lib/elixir/lib/code/typespec.ex b/lib/elixir/lib/code/typespec.ex index 7cda6db78b0..cd38949bc70 100644 --- a/lib/elixir/lib/code/typespec.ex +++ b/lib/elixir/lib/code/typespec.ex @@ -175,7 +175,8 @@ defmodule Code.Typespec do defp get_module_and_beam(module) when is_atom(module) do with {^module, beam, _filename} <- :code.get_object_code(module), - {:ok, ^module} <- beam |> :beam_lib.info() |> Keyword.fetch(:module) do + info_pairs when is_list(info_pairs) <- :beam_lib.info(beam), + {:ok, ^module} <- Keyword.fetch(info_pairs, :module) do {module, beam} else _ -> :error diff --git a/lib/iex/lib/iex/helpers.ex b/lib/iex/lib/iex/helpers.ex index f693113eb11..fa498735621 100644 --- a/lib/iex/lib/iex/helpers.ex +++ b/lib/iex/lib/iex/helpers.ex @@ -1476,7 +1476,8 @@ defmodule IEx.Helpers do defp get_beam_and_path(module) do with {^module, beam, filename} <- :code.get_object_code(module), - {:ok, ^module} <- beam |> :beam_lib.info() |> Keyword.fetch(:module) do + info_pairs when is_list(info_pairs) <- :beam_lib.info(beam), + {:ok, ^module} <- Keyword.fetch(info_pairs, :module) do {beam, filename} else _ -> :error diff --git a/lib/iex/lib/iex/info.ex b/lib/iex/lib/iex/info.ex index 2878f53807e..b1b9c0fae8c 100644 --- a/lib/iex/lib/iex/info.ex +++ b/lib/iex/lib/iex/info.ex @@ -89,7 +89,7 @@ defimpl IEx.Info, for: Atom do {^atom, beam, _path} -> info = :beam_lib.info(beam) - Keyword.fetch(info, :module) == {:ok, atom} + is_list(info) and Keyword.fetch(info, :module) == {:ok, atom} end end From 9b480f985bcd7acdcfd2886ae1e31567f4ae2925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Andr=C3=A9=20Jakobsen?= Date: Mon, 6 Nov 2023 13:27:14 +0100 Subject: [PATCH 033/185] [docs] Clean up sigils intro (#13077) --- lib/elixir/pages/getting-started/sigils.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/getting-started/sigils.md b/lib/elixir/pages/getting-started/sigils.md index 7897f277a4c..9da5509a42d 100644 --- a/lib/elixir/pages/getting-started/sigils.md +++ b/lib/elixir/pages/getting-started/sigils.md @@ -2,7 +2,7 @@ Elixir provides double-quoted strings as well as a concept called charlists, which are defined using the `~c"hello world"` sigil syntax. In this chapter, we will learn more about sigils and how to define our own. -One of Elixir's goals is extensibility: developers should be able to extend the language to fit any particular domain. Sigils provide the foundation for extending the language with custom textual representations. Sigils start with the tilde (`~`) character which is followed by a one lower-case letter or several upper-case ones and then a delimiter. Optionally, modifiers can be added after the final delimiter. +One of Elixir's goals is extensibility: developers should be able to extend the language to fit any particular domain. Sigils provide the foundation for extending the language with custom textual representations. Sigils start with the tilde (`~`) character which is followed by either a single lower-case letter or one or more upper-case letters, and then a delimiter. Optional modifiers are added after the final delimiter. ## Regular expressions From 8920cc143208f53ba0c8d73335f5876e3a35567d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 6 Nov 2023 13:28:49 +0100 Subject: [PATCH 034/185] Fix case clause error on tokenizer --- lib/elixir/src/elixir_tokenizer.erl | 2 +- lib/elixir/test/elixir/kernel/parser_test.exs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 6db3de8c6ec..951b4b55abf 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -794,7 +794,7 @@ handle_strings(T, Line, Column, H, Scope, Tokens) -> Token = {kw_identifier, {Line, Column - 1, nil}, Atom}, tokenize(Rest, NewLine, NewColumn + 1, NewScope, [Token | Tokens]); {error, Reason} -> - {error, Reason, Rest, Tokens} + error(Reason, Rest, NewScope, Tokens) end; {ok, Unescaped} -> diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index cc9e0531574..149498ec2e3 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -1136,7 +1136,12 @@ defmodule Kernel.ParserTest do assert_syntax_error( ["atom length must be less than system limit: "], - ~s[:"#{atom}"] + ~s{:"#{atom}"} + ) + + assert_syntax_error( + ["atom length must be less than system limit: "], + ~s{["#{atom}": 123]} ) end end From 33e1b570bf7eca81651937230edb7e6492603ca5 Mon Sep 17 00:00:00 2001 From: Lucas Francisco da Matta Vegi Date: Mon, 6 Nov 2023 11:37:48 -0300 Subject: [PATCH 035/185] Additional remarks for maintaining research history (#13078) Similar to what we had already done with other anti-patterns that changed names --- lib/elixir/pages/anti-patterns/design-anti-patterns.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index 4f94b6d159b..aedc3a7a53d 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -186,6 +186,10 @@ end A common practice followed by the community is to make the non-raising version return `{:ok, result}` or `{:error, Exception.t}`. For example, an HTTP client may return `{:ok, %HTTP.Response{}}` on success cases and `{:error, %HTTP.Error{}}` for failures, where `HTTP.Error` is [implemented as an exception](`Kernel.defexception/1`). This makes it convenient for anyone to raise an exception by simply calling `Kernel.raise/1`. +#### Additional remarks + +This anti-pattern was formerly known as [Using exceptions for control-flow](https://github.com/lucasvegi/Elixir-Code-Smells#using-exceptions-for-control-flow). + ## Primitive obsession #### Problem @@ -294,6 +298,10 @@ The following arguments were given to MyLibrary.foo/1: my_library.ex:2: MyLibrary.foo/1 ``` +#### Additional remarks + +This anti-pattern was formerly known as [Working with invalid data](https://github.com/lucasvegi/Elixir-Code-Smells#working-with-invalid-data). + ## Unrelated multi-clause function #### Problem From 7d4d42097da270f6169916d34de20ecbb1925dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 6 Nov 2023 15:51:37 +0100 Subject: [PATCH 036/185] Restore code paths in archive.install/escript.install Closes #13079. --- lib/mix/lib/mix/local/installer.ex | 79 +++++++++++++----------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/lib/mix/lib/mix/local/installer.ex b/lib/mix/lib/mix/local/installer.ex index ea39b6d78fd..12fbd41e21f 100644 --- a/lib/mix/lib/mix/local/installer.ex +++ b/lib/mix/lib/mix/local/installer.ex @@ -319,8 +319,15 @@ defmodule Mix.Local.Installer do """ @spec fetch(tuple, (atom -> any), (atom -> any)) :: any def fetch(dep_spec, in_fetcher \\ &in_fetcher/1, in_package) do - with_tmp_dir(fn tmp_path -> + tmp_path = tmp_path() + previous_env = Mix.env() + deps_path = System.get_env("MIX_DEPS_PATH") + code_path = :code.get_path() + + try do File.mkdir_p!(tmp_path) + System.delete_env("MIX_DEPS_PATH") + Mix.env(:prod) File.write!(Path.join(tmp_path, "mix.exs"), """ defmodule Mix.Local.Installer.MixProject do @@ -336,34 +343,36 @@ defmodule Mix.Local.Installer do end """) - with_reset_prod_env(fn -> - Mix.ProjectStack.on_clean_slate(fn -> - tmp_path = - Mix.Project.in_project(:mix_local_installer, tmp_path, [], fn mix_exs -> - in_fetcher.(mix_exs) + Mix.ProjectStack.on_clean_slate(fn -> + tmp_path = + Mix.Project.in_project(:mix_local_installer, tmp_path, [], fn mix_exs -> + in_fetcher.(mix_exs) - # The tmp_dir may have symlinks in it, so we properly resolve - # the directory before customizing deps_path and lockfile. - File.cwd!() - end) + # The tmp_dir may have symlinks in it, so we properly resolve + # the directory before customizing deps_path and lockfile. + File.cwd!() + end) - {package_name, package_path} = package_name_path(dep_spec, tmp_path) + {package_name, package_path} = package_name_path(dep_spec, tmp_path) - post_config = [ - deps_path: Path.join(tmp_path, "deps"), - lockfile: Path.join(tmp_path, "mix.lock") - ] + post_config = [ + deps_path: Path.join(tmp_path, "deps"), + lockfile: Path.join(tmp_path, "mix.lock") + ] - Mix.Project.in_project(package_name, package_path, post_config, fn mix_exs -> - in_fetcher.(mix_exs) - in_package.(mix_exs) - end) + Mix.Project.in_project(package_name, package_path, post_config, fn mix_exs -> + in_fetcher.(mix_exs) + in_package.(mix_exs) end) end) - end) - after - :code.purge(Mix.Local.Installer.Fetcher) - :code.delete(Mix.Local.Installer.Fetcher) + after + File.rm_rf(tmp_path) + Mix.env(previous_env) + deps_path && System.put_env("MIX_DEPS_PATH", deps_path) + :code.set_path(code_path) + :code.purge(Mix.Local.Installer.Fetcher) + :code.delete(Mix.Local.Installer.Fetcher) + end end defp package_name_path(dep_spec, tmp_path) do @@ -388,28 +397,8 @@ defmodule Mix.Local.Installer do Mix.Task.run("deps.get", ["--only", Atom.to_string(Mix.env())]) end - defp with_tmp_dir(fun) do + defp tmp_path do unique = :crypto.strong_rand_bytes(4) |> Base.url_encode64(padding: false) - tmp_path = Path.join(System.tmp_dir!(), "mix-local-installer-fetcher-" <> unique) - - try do - fun.(tmp_path) - after - File.rm_rf(tmp_path) - end - end - - defp with_reset_prod_env(fun) do - previous_env = Mix.env() - deps_path = System.get_env("MIX_DEPS_PATH") - - try do - System.delete_env("MIX_DEPS_PATH") - Mix.env(:prod) - fun.() - after - Mix.env(previous_env) - deps_path && System.put_env("MIX_DEPS_PATH", deps_path) - end + Path.join(System.tmp_dir!(), "mix-local-installer-fetcher-" <> unique) end end From 7ebf5c3032c19cdeed52050edb31184a44883560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 6 Nov 2023 16:30:46 +0100 Subject: [PATCH 037/185] Add :emit_warnings to Code.string_to_quoted --- lib/elixir/lib/code.ex | 8 +++----- lib/elixir/lib/code/fragment.ex | 2 +- lib/elixir/src/elixir.erl | 6 +++--- lib/elixir/src/elixir.hrl | 1 - lib/elixir/src/elixir_tokenizer.erl | 5 +---- lib/mix/lib/mix/dep/lock.ex | 2 +- 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index a700b71b674..2d6a0d02f11 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -962,10 +962,9 @@ defmodule Code do to_quoted_opts = [ unescape: false, - warn_on_unnecessary_quotes: false, literal_encoder: &{:ok, {:__block__, &2, [&1]}}, token_metadata: true, - warnings: false + emit_warnings: false ] ++ opts {forms, comments} = string_to_quoted_with_comments!(string, to_quoted_opts) @@ -1126,9 +1125,8 @@ defmodule Code do atoms but `:existing_atoms_only` is still used for dynamic atoms, such as atoms with interpolations. - * `:warn_on_unnecessary_quotes` - when `false`, does not warn - when atoms, keywords or calls have unnecessary quotes on - them. Defaults to `true`. + * `:emit_warnings` (since v1.16.0) - when `false`, does not emit + tokenizing/parsing related warnings. Defaults to `true`. ## `Macro.to_string/2` diff --git a/lib/elixir/lib/code/fragment.ex b/lib/elixir/lib/code/fragment.ex index ace8d467661..a95f0821902 100644 --- a/lib/elixir/lib/code/fragment.ex +++ b/lib/elixir/lib/code/fragment.ex @@ -1107,6 +1107,6 @@ defmodule Code.Fragment do opts = Keyword.take(opts, [:file, :line, :column, :columns, :token_metadata, :literal_encoder]) - Code.string_to_quoted(fragment, [cursor_completion: true, warnings: false] ++ opts) + Code.string_to_quoted(fragment, [cursor_completion: true, emit_warnings: false] ++ opts) end end diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl index a7a1e7451e6..f4ff932922e 100644 --- a/lib/elixir/src/elixir.erl +++ b/lib/elixir/src/elixir.erl @@ -486,7 +486,7 @@ string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(Sta {ok, _Line, _Column, [], Tokens} -> {ok, Tokens}; {ok, _Line, _Column, Warnings, Tokens} -> - (lists:keyfind(warnings, 1, Opts) /= {warnings, false}) andalso + (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)], {ok, Tokens}; {error, {Location, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _Warnings, _SoFar} -> @@ -544,8 +544,8 @@ to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom). handle_parsing_opts(File, Opts) -> WarningFile = - case lists:keyfind(warnings, 1, Opts) of - {warnings, false} -> nil; + case lists:keyfind(emit_warnings, 1, Opts) of + {emit_warnings, false} -> nil; _ -> File end, LiteralEncoder = diff --git a/lib/elixir/src/elixir.hrl b/lib/elixir/src/elixir.hrl index 3199ea2ebea..e5d12c93386 100644 --- a/lib/elixir/src/elixir.hrl +++ b/lib/elixir/src/elixir.hrl @@ -36,6 +36,5 @@ ascii_identifiers_only=true, indentation=0, mismatch_hints=[], - warn_on_unnecessary_quotes=true, warnings=[] }). diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 951b4b55abf..144fdc39cfb 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -123,8 +123,6 @@ tokenize(String, Line, Column, Opts) -> Acc#elixir_tokenizer{preserve_comments=PreserveComments}; ({unescape, Unescape}, Acc) when is_boolean(Unescape) -> Acc#elixir_tokenizer{unescape=Unescape}; - ({warn_on_unnecessary_quotes, Unnecessary}, Acc) when is_boolean(Unnecessary) -> - Acc#elixir_tokenizer{warn_on_unnecessary_quotes=Unnecessary}; (_, Acc) -> Acc end, #elixir_tokenizer{identifier_tokenizer=IdentifierTokenizer}, Opts), @@ -964,12 +962,11 @@ eol(_Line, _Column, [{eol, {Line, Column, Count}} | Tokens]) -> eol(Line, Column, Tokens) -> [{eol, {Line, Column, 1}} | Tokens]. -is_unnecessary_quote([Part], #elixir_tokenizer{warn_on_unnecessary_quotes=true} = Scope) when is_list(Part) -> +is_unnecessary_quote([Part], Scope) when is_list(Part) -> case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(Part) of {identifier, _, [], _, true, Special} -> not lists:member(at, Special); _ -> false end; - is_unnecessary_quote(_Parts, _Scope) -> false. diff --git a/lib/mix/lib/mix/dep/lock.ex b/lib/mix/lib/mix/dep/lock.ex index c104e26f345..248831caf0a 100644 --- a/lib/mix/lib/mix/dep/lock.ex +++ b/lib/mix/lib/mix/dep/lock.ex @@ -11,7 +11,7 @@ defmodule Mix.Dep.Lock do """ @spec read(Path.t()) :: map() def read(lockfile \\ lockfile()) do - opts = [file: lockfile, warn_on_unnecessary_quotes: false] + opts = [file: lockfile, emit_warnings: false] with {:ok, contents} <- File.read(lockfile), assert_no_merge_conflicts_in_lockfile(lockfile, contents), From 37f88322297a3b5b4cfb32a997d036900981a063 Mon Sep 17 00:00:00 2001 From: Jacob Swanner Date: Tue, 7 Nov 2023 00:59:24 -0800 Subject: [PATCH 038/185] Fix Enum cheatsheet for drop/2 and take/2 with negative index (#13080) --- lib/elixir/pages/cheatsheets/enum-cheat.cheatmd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/elixir/pages/cheatsheets/enum-cheat.cheatmd b/lib/elixir/pages/cheatsheets/enum-cheat.cheatmd index 1e6190b62ad..4d5a63ea0f6 100644 --- a/lib/elixir/pages/cheatsheets/enum-cheat.cheatmd +++ b/lib/elixir/pages/cheatsheets/enum-cheat.cheatmd @@ -820,7 +820,10 @@ Negative indexes count from the back: ```elixir iex> Enum.drop(cart, -1) -[%{fruit: "orange", count: 6}] +[ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} +] ``` ### [`drop_every(enum, nth)`](`Enum.drop_every/2`) @@ -851,10 +854,7 @@ Negative indexes count from the back: ```elixir iex> Enum.take(cart, -1) -[ - %{fruit: "apple", count: 3}, - %{fruit: "banana", count: 1} -] +[%{fruit: "orange", count: 6}] ``` ### [`take_every(enum, nth)`](`Enum.take_every/2`) From 7be85838fad14a1058652715f6f4fa4d1df43e36 Mon Sep 17 00:00:00 2001 From: Juan Barrios <03juan@users.noreply.github.com> Date: Tue, 7 Nov 2023 21:10:50 +0200 Subject: [PATCH 039/185] Update float.ex description of ceil/2 and floor/2 (#13084) --- lib/elixir/lib/float.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/float.ex b/lib/elixir/lib/float.ex index 49862329e6f..62b5dad25f8 100644 --- a/lib/elixir/lib/float.ex +++ b/lib/elixir/lib/float.ex @@ -198,7 +198,7 @@ defmodule Float do defp add_dot(acc, false), do: acc <> ".0" @doc """ - Rounds a float to the largest number less than or equal to `num`. + Rounds a float to the largest float less than or equal to `number`. `floor/2` also accepts a precision to round a floating-point value down to an arbitrary number of fractional digits (between 0 and 15). @@ -246,7 +246,7 @@ defmodule Float do end @doc """ - Rounds a float to the smallest integer greater than or equal to `num`. + Rounds a float to the smallest float greater than or equal to `number`. `ceil/2` also accepts a precision to round a floating-point value down to an arbitrary number of fractional digits (between 0 and 15). From b9fffa3b41661dcf981a38c7522b7488047396cf Mon Sep 17 00:00:00 2001 From: Minh Dao <43783196+minhqdao@users.noreply.github.com> Date: Tue, 7 Nov 2023 22:51:35 +0100 Subject: [PATCH 040/185] Fix typo in getting-started guide (#13085) Co-authored-by: minhqdao --- lib/elixir/pages/getting-started/basic-types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/getting-started/basic-types.md b/lib/elixir/pages/getting-started/basic-types.md index 140192e214c..6b26fba71c7 100644 --- a/lib/elixir/pages/getting-started/basic-types.md +++ b/lib/elixir/pages/getting-started/basic-types.md @@ -83,7 +83,7 @@ You can also use `is_float` or `is_number` to check, respectively, if an argumen ## Identifying functions and documentation -Before we move on to the next data type, let's talk about how Elixir identity functions. +Before we move on to the next data type, let's talk about how Elixir identifies functions. Functions in Elixir are identified by both their name and their arity. The arity of a function describes the number of arguments that the function takes. From this point on we will use both the function name and its arity to describe functions throughout the documentation. `trunc/1` identifies the function which is named `trunc` and takes `1` argument, whereas `trunc/2` identifies a different (nonexistent) function with the same name but with an arity of `2`. From 033a177078b062e4d5a85007c7189259092e6201 Mon Sep 17 00:00:00 2001 From: Cameron Duley Date: Thu, 9 Nov 2023 15:13:03 -0500 Subject: [PATCH 041/185] Fix `String.replace_invalid/2` perf regressions (#13090) --- lib/elixir/lib/string.ex | 112 +++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 58 deletions(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index 5bfadfd5627..fe47e33dd78 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -1871,6 +1871,22 @@ defmodule String do end end + defguardp replace_invalid_ii_of_iii(i, ii) + when (896 <= Bitwise.bor(Bitwise.bsl(i, 6), ii) and + Bitwise.bor(Bitwise.bsl(i, 6), ii) <= 1023) or + (32 <= Bitwise.bor(Bitwise.bsl(i, 6), ii) and + Bitwise.bor(Bitwise.bsl(i, 6), ii) <= 863) + + defguardp replace_invalid_ii_of_iv(i, ii) + when 16 <= Bitwise.bor(Bitwise.bsl(i, 6), ii) and + Bitwise.bor(Bitwise.bsl(i, 6), ii) <= 271 + + defguardp replace_invalid_iii_of_iv(i, ii, iii) + when 1024 <= Bitwise.bor(Bitwise.bor(Bitwise.bsl(i, 12), Bitwise.bsl(ii, 6)), iii) and + Bitwise.bor(Bitwise.bor(Bitwise.bsl(i, 12), Bitwise.bsl(ii, 6)), iii) <= 17407 + + defguardp replace_invalid_is_next(next) when Bitwise.bsr(next, 6) !== 0b10 + @doc ~S""" Returns a new string created by replacing all invalid bytes with `replacement` (`"�"` by default). @@ -1889,94 +1905,74 @@ defmodule String do "nem rán bERROR! bề" """ @doc since: "1.16.0" - def replace_invalid(string, replacement \\ "�") - when is_binary(string) and is_binary(replacement) do - do_replace_invalid(string, replacement, <<>>) + def replace_invalid(bytes, replacement \\ "�") + when is_binary(bytes) and is_binary(replacement) do + do_replace_invalid(bytes, replacement, <<>>) end # Valid ASCII (for better average speed) - defp do_replace_invalid(<>, rep, acc) - when ascii in 0..127 and n_lead != 0b10 do - do_replace_invalid(<>, rep, <>) + defp do_replace_invalid(<> = rest, rep, acc) + when ascii in 0..127 and replace_invalid_is_next(next) do + <<_::8, rest::bytes>> = rest + do_replace_invalid(rest, rep, acc <> <>) end # Valid UTF-8 - defp do_replace_invalid(<>, rep, acc) do - do_replace_invalid(rest, rep, <>) + defp do_replace_invalid(<>, rep, acc) do + do_replace_invalid(rest, rep, acc <> <>) end # 2/3 truncated sequence - defp do_replace_invalid(<<0b1110::4, i::4, 0b10::2, ii::6>>, rep, acc) do - <> = <> - <> + defp do_replace_invalid(<<0b1110::4, i::4, 0b10::2, ii::6>>, rep, acc) + when replace_invalid_ii_of_iii(i, ii) do + acc <> rep end - defp do_replace_invalid(<<0b1110::4, i::4, 0b10::2, ii::6, n_lead::2, rest::bits>>, rep, acc) - when n_lead != 0b10 do - <> = <> - - do_replace_invalid( - <>, - rep, - <> - ) + defp do_replace_invalid(<<0b1110::4, i::4, 0b10::2, ii::6, next::8, _::bytes>> = rest, rep, acc) + when replace_invalid_ii_of_iii(i, ii) and replace_invalid_is_next(next) do + <<_::16, rest::bytes>> = rest + do_replace_invalid(rest, rep, acc <> rep) end # 2/4 - defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6>>, rep, acc) do - <> = <> - <> + defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6>>, rep, acc) + when replace_invalid_ii_of_iv(i, ii) do + acc <> rep end - defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6, n_lead::2, rest::bits>>, rep, acc) - when n_lead != 0b10 do - <> = <> - - do_replace_invalid( - <>, - rep, - <> - ) + defp do_replace_invalid( + <<0b11110::5, i::3, 0b10::2, ii::6, next::8, _::bytes>> = rest, + rep, + acc + ) + when replace_invalid_ii_of_iv(i, ii) and replace_invalid_is_next(next) do + <<_::16, rest::bytes>> = rest + do_replace_invalid(rest, rep, acc <> rep) end # 3/4 - defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6, 0b10::2, iii::6>>, rep, acc) do - <> = <> - <> + defp do_replace_invalid(<<0b11110::5, i::3, 0b10::2, ii::6, 0b10::2, iii::6>>, rep, acc) + when replace_invalid_iii_of_iv(i, ii, iii) do + acc <> rep end defp do_replace_invalid( - <<0b11110::5, i::3, 0b10::2, ii::6, 0b10::2, iii::6, n_lead::2, rest::bits>>, + <<0b11110::5, i::3, 0b10::2, ii::6, 0b10::2, iii::6, next::8, _::bytes>> = rest, rep, acc ) - when n_lead != 0b10 do - <> = <> - - do_replace_invalid( - <>, - rep, - <> - ) + when replace_invalid_iii_of_iv(i, ii, iii) and replace_invalid_is_next(next) do + <<_::24, rest::bytes>> = rest + do_replace_invalid(rest, rep, acc <> rep) end - # any other invalid bytes - defp do_replace_invalid(<<_, rest::bits>>, rep, acc), - do: do_replace_invalid(rest, rep, <>) + # Everything else + defp do_replace_invalid(<<_, rest::bytes>>, rep, acc), + do: do_replace_invalid(rest, rep, acc <> rep) + # Final defp do_replace_invalid(<<>>, _, acc), do: acc - # bounds-checking truncated code points for overlong encodings - defp replace_invalid_ii_of_iii(tcp, rep) when tcp >= 32 and tcp <= 863, do: rep - defp replace_invalid_ii_of_iii(tcp, rep) when tcp >= 896 and tcp <= 1023, do: rep - defp replace_invalid_ii_of_iii(_, rep), do: rep <> rep - - defp replace_invalid_ii_of_iiii(tcp, rep) when tcp >= 16 and tcp <= 271, do: rep - defp replace_invalid_ii_of_iiii(_, rep), do: rep <> rep - - defp replace_invalid_iii_of_iiii(tcp, rep) when tcp >= 1024 and tcp <= 17407, do: rep - defp replace_invalid_iii_of_iiii(_, rep), do: rep <> rep <> rep - @doc ~S""" Splits the string into chunks of characters that share a common trait. From 56e6494b36458ba0b3ffd4bf4ade1ec13d0e96ea Mon Sep 17 00:00:00 2001 From: Cameron Duley Date: Thu, 9 Nov 2023 18:59:05 -0500 Subject: [PATCH 042/185] Use `in/2` in `String.replace_invalid/2` guards (#13093) --- lib/elixir/lib/string.ex | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index fe47e33dd78..1966c533409 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -1872,18 +1872,14 @@ defmodule String do end defguardp replace_invalid_ii_of_iii(i, ii) - when (896 <= Bitwise.bor(Bitwise.bsl(i, 6), ii) and - Bitwise.bor(Bitwise.bsl(i, 6), ii) <= 1023) or - (32 <= Bitwise.bor(Bitwise.bsl(i, 6), ii) and - Bitwise.bor(Bitwise.bsl(i, 6), ii) <= 863) + when Bitwise.bor(Bitwise.bsl(i, 6), ii) in 32..863 or + Bitwise.bor(Bitwise.bsl(i, 6), ii) in 896..1023 defguardp replace_invalid_ii_of_iv(i, ii) - when 16 <= Bitwise.bor(Bitwise.bsl(i, 6), ii) and - Bitwise.bor(Bitwise.bsl(i, 6), ii) <= 271 + when Bitwise.bor(Bitwise.bsl(i, 6), ii) in 16..271 defguardp replace_invalid_iii_of_iv(i, ii, iii) - when 1024 <= Bitwise.bor(Bitwise.bor(Bitwise.bsl(i, 12), Bitwise.bsl(ii, 6)), iii) and - Bitwise.bor(Bitwise.bor(Bitwise.bsl(i, 12), Bitwise.bsl(ii, 6)), iii) <= 17407 + when Bitwise.bor(Bitwise.bor(Bitwise.bsl(i, 12), Bitwise.bsl(ii, 6)), iii) in 1024..17407 defguardp replace_invalid_is_next(next) when Bitwise.bsr(next, 6) !== 0b10 From 2a94668e6e89f2f90953e5167041586de26a5998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 10 Nov 2023 12:36:04 +0100 Subject: [PATCH 043/185] Let's not deprecate ...foo as the API may be useful for the type system in the future. This reverts commit f97d8585e8c7c1c6d578d927c0cda47eb9ef79b4. --- lib/elixir/src/elixir_parser.yrl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index de678104fd2..ce6afd32172 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -910,11 +910,6 @@ build_identifier({'.', Meta, _} = Dot, Args) -> build_identifier({op_identifier, Location, Identifier}, [Arg]) -> {Identifier, [{ambiguous_op, nil} | meta_from_location(Location)], [Arg]}; -%% TODO: Either remove ... or make it an operator on v2.0 -build_identifier({_, {Line, Column, _} = Location, '...'}, Args) when is_list(Args) -> - warn({Line, Column}, "... is no longer supported as a function call and it must receive no arguments"), - {'...', meta_from_location(Location), Args}; - build_identifier({_, Location, Identifier}, Args) -> {Identifier, meta_from_location(Location), Args}. From beaa33d08885e21eb79bc0707b1a8db3e294e9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 10 Nov 2023 12:53:34 +0100 Subject: [PATCH 044/185] Handle nil values in IO.warn --- lib/elixir/src/elixir_errors.erl | 6 +- .../test/elixir/kernel/diagnostics_test.exs | 100 ++++-------------- 2 files changed, 23 insertions(+), 83 deletions(-) diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index a7969afb1df..e3212502da1 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -57,8 +57,7 @@ get_snippet(File, Position) -> LineNumber = extract_line(Position), get_file_line(File, LineNumber). -get_file_line(_, 0) -> nil; -get_file_line(File, LineNumber) -> +get_file_line(File, LineNumber) when is_integer(LineNumber), LineNumber > 0 -> case file:open(File, [read, binary]) of {ok, IoDevice} -> Line = traverse_file_line(IoDevice, LineNumber), @@ -66,7 +65,8 @@ get_file_line(File, LineNumber) -> Line; {error, _} -> nil - end. + end; +get_file_line(_, _) -> nil. traverse_file_line(IoDevice, 1) -> case file:read_line(IoDevice) of diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index 6e30f0f10b4..a34dbb4e1d0 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -544,101 +544,41 @@ defmodule Kernel.DiagnosticsTest do end @tag :tmp_dir - test "long message (file)", %{tmp_dir: tmp_dir} do - path = make_relative_tmp(tmp_dir, "long-warning.ex") + test "IO.warn file+line+column", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "io-warn-file-line-column.ex") source = """ - defmodule Sample do - @file "#{path}" - - def atom_case do - v = "bc" - - case v do - _ when is_atom(v) -> :ok - _ -> :fail - end - end - end + IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line, column: 4) """ File.write!(path, source) expected = """ - warning: incompatible types: - - binary() !~ atom() - - in expression: - - # #{path}:8 - is_atom(v) - - where "v" was given the type binary() in: - - # #{path}:5 - v = "bc" - - where "v" was given the type atom() in: - - # #{path}:8 - is_atom(v) - - Conflict found at + warning: oops + multi + line │ - 8 │ _ when is_atom(v) -> :ok - │ ~ + 1 │ IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line, column: 4) + │ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ │ - └─ #{path}:8:14: Sample.atom_case/0 + └─ tmp\ """ - assert capture_eval(source) =~ expected - after - purge(Sample) + assert capture_io(:stderr, fn -> Code.eval_file(path) end) =~ expected end - test "long message (nofile)" do - source = """ - defmodule Sample do - def atom_case do - v = "bc" - - case v do - _ when is_atom(v) -> :ok - _ -> :fail - end - end - end - """ - - expected = """ - warning: incompatible types: - - binary() !~ atom() - - in expression: - - # nofile:6 - is_atom(v) + test "IO.warn with missing data" do + assert capture_eval(""" + IO.warn("oops-bad", file: #{inspect(__ENV__.file)}, line: 3, column: nil) + """) =~ "warning: oops-bad" - where "v" was given the type binary() in: + assert capture_eval(""" + IO.warn("oops-bad", file: #{inspect(__ENV__.file)}, line: nil) + """) =~ "oops-bad" - # nofile:3 - v = "bc" - - where "v" was given the type atom() in: - - # nofile:6 - is_atom(v) - - Conflict found at - └─ nofile:6:14: Sample.atom_case/0 - - """ - - assert capture_eval(source) =~ expected - after - purge(Sample) + assert capture_eval(""" + IO.warn("oops-bad", file: nil) + """) =~ "oops-bad" end @tag :tmp_dir From 0a146d10cccfb5730cfa4e73ee3ea1cad540c884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 11 Nov 2023 08:59:55 +0100 Subject: [PATCH 045/185] Fix link, closes #13095 --- lib/elixir/pages/mix-and-otp/agents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/mix-and-otp/agents.md b/lib/elixir/pages/mix-and-otp/agents.md index 935064a4923..5c9026eebb9 100644 --- a/lib/elixir/pages/mix-and-otp/agents.md +++ b/lib/elixir/pages/mix-and-otp/agents.md @@ -124,7 +124,7 @@ The first step in our implementation is to call `use Agent`. Most of the functio Then we define a `start_link/1` function, which will effectively start the agent. It is a convention to define a `start_link/1` function that always accepts a list of options. We don't plan on using any options right now, but we might later on. We then proceed to call `Agent.start_link/1`, which receives an anonymous function that returns the Agent's initial state. -We are keeping a map inside the agent to store our keys and values. Getting and putting values on the map is done with the Agent API and the capture operator `&`, introduced in [the Getting Started guide](../getting-started/modules-and-functions.md#function-capturing). The agent passes its state to the anonymous function via the `&1` argument when `Agent.get/2` and `Agent.update/2` are called. +We are keeping a map inside the agent to store our keys and values. Getting and putting values on the map is done with the Agent API and the capture operator `&`, introduced in [the Getting Started guide](../getting-started/anonymous-functions.md#the-capture-operator). The agent passes its state to the anonymous function via the `&1` argument when `Agent.get/2` and `Agent.update/2` are called. Now that the `KV.Bucket` module has been defined, our test should pass! You can try it yourself by running: `mix test`. From 68782004a764a844556d95f91510e744de1c94f6 Mon Sep 17 00:00:00 2001 From: Christopher Keele Date: Sat, 11 Nov 2023 02:20:01 -0600 Subject: [PATCH 046/185] Produce better error messages for non-binary mix git deps refspecs. (#13088) When using git dependencies, a branch/ref/tag specifier is passed verbatim to `System.cmd/3`. This can lead to intimidating error messages when they are not provided as a binary (for instance, an atom like `tag: :stable`): ``` ** (ArgumentError) all arguments for System.cmd/3 must be binaries (elixir 1.15.6) lib/system.ex:1083: System.cmd/3 (mix 1.15.6) lib/mix/scm/git.ex:287: Mix.SCM.Git.git!/2 ``` This PR adds a check during git opts verification time to provide better feedback. --- lib/mix/lib/mix/scm/git.ex | 26 ++++++++++++++++---------- lib/mix/test/mix/scm/git_test.exs | 16 +++++++++++++++- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/lib/mix/lib/mix/scm/git.ex b/lib/mix/lib/mix/scm/git.ex index 7b46434443a..a5471bca189 100644 --- a/lib/mix/lib/mix/scm/git.ex +++ b/lib/mix/lib/mix/scm/git.ex @@ -201,18 +201,24 @@ defmodule Mix.SCM.Git do ## Helpers defp validate_git_options(opts) do - err = - "You should specify only one of branch, ref or tag, and only once. " <> - "Error on Git dependency: #{redact_uri(opts[:git])}" + case Keyword.take(opts, [:branch, :ref, :tag]) do + [] -> + opts - validate_single_uniq(opts, [:branch, :ref, :tag], err) - end + [{_refspec, value}] when is_binary(value) -> + opts - defp validate_single_uniq(opts, take, error) do - case Keyword.take(opts, take) do - [] -> opts - [_] -> opts - _ -> Mix.raise(error) + [{refspec, value}] -> + Mix.raise( + "A dependency's #{refspec} must be a string, got: #{inspect(value)}. " <> + "Error on Git dependency: #{redact_uri(opts[:git])}" + ) + + _ -> + Mix.raise( + "You should specify only one of branch, ref or tag, and only once. " <> + "Error on Git dependency: #{redact_uri(opts[:git])}" + ) end end diff --git a/lib/mix/test/mix/scm/git_test.exs b/lib/mix/test/mix/scm/git_test.exs index 96634bc355c..24429510544 100644 --- a/lib/mix/test/mix/scm/git_test.exs +++ b/lib/mix/test/mix/scm/git_test.exs @@ -50,7 +50,7 @@ defmodule Mix.SCM.GitTest do end end - test "raises about conflicting Git checkout options" do + test "raises about conflicting Git refspec options" do assert_raise Mix.Error, ~r/You should specify only one of branch, ref or tag/, fn -> Mix.SCM.Git.accepts_options(nil, git: "/repo", branch: "main", tag: "0.1.0") end @@ -60,6 +60,20 @@ defmodule Mix.SCM.GitTest do end end + test "raises about non-binary Git refspec options" do + assert_raise Mix.Error, ~r/A dependency's branch must be a string/, fn -> + Mix.SCM.Git.accepts_options(nil, git: "/repo", branch: :main) + end + + assert_raise Mix.Error, ~r/A dependency's tag must be a string/, fn -> + Mix.SCM.Git.accepts_options(nil, git: "/repo", tag: :stable) + end + + assert_raise Mix.Error, ~r/A dependency's ref must be a string/, fn -> + Mix.SCM.Git.accepts_options(nil, git: "/repo", ref: :abcdef0123456789) + end + end + defp lock(opts \\ []) do [lock: {:git, "/repo", "abcdef0123456789", opts}] end From 8f28bd9c3f6edf254fd503fe22cbb3cc0d0fc317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 11 Nov 2023 21:55:18 +0100 Subject: [PATCH 047/185] Fix GenServer cheatsheet link Closes #13098. --- lib/elixir/pages/mix-and-otp/genservers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/mix-and-otp/genservers.md b/lib/elixir/pages/mix-and-otp/genservers.md index 51be7cbe5b3..74da0577c36 100644 --- a/lib/elixir/pages/mix-and-otp/genservers.md +++ b/lib/elixir/pages/mix-and-otp/genservers.md @@ -320,7 +320,7 @@ So far we have used three callbacks: `handle_call/3`, `handle_cast/2` and `handl Since any message, including the ones sent via `send/2`, go to `handle_info/2`, there is a chance that unexpected messages will arrive to the server. Therefore, if we don't define the catch-all clause, those messages could cause our registry to crash, because no clause would match. We don't need to worry about such cases for `handle_call/3` and `handle_cast/2` though. Calls and casts are only done via the `GenServer` API, so an unknown message is quite likely a developer mistake. -To help developers remember the differences between call, cast and info, the supported return values and more, we have a tiny [GenServer cheat sheet](/downloads/cheatsheets/gen-server.pdf). +To help developers remember the differences between call, cast and info, the supported return values and more, we have a tiny [GenServer cheat sheet](https://elixir-lang.org/downloads/cheatsheets/gen-server.pdf). ## Monitors or links? From abbf9061ea23d7692ef1cea565fed439bd5dff69 Mon Sep 17 00:00:00 2001 From: Artem Solomatin Date: Sun, 12 Nov 2023 12:07:12 +0300 Subject: [PATCH 048/185] Fix links references (#13099) --- lib/elixir/pages/getting-started/keywords-and-maps.md | 2 +- lib/elixir/pages/mix-and-otp/distributed-tasks.md | 2 +- lib/elixir/pages/mix-and-otp/genservers.md | 2 +- lib/elixir/pages/references/syntax-reference.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/elixir/pages/getting-started/keywords-and-maps.md b/lib/elixir/pages/getting-started/keywords-and-maps.md index e99b38399f3..11b1c01cbf1 100644 --- a/lib/elixir/pages/getting-started/keywords-and-maps.md +++ b/lib/elixir/pages/getting-started/keywords-and-maps.md @@ -127,7 +127,7 @@ iex> if true, do: "This will be seen", else: "This won't" Pay close attention to both syntaxes. In the keyword list format, we separate each key-value pair with commas, and each key is followed by `:`. In the `do`-blocks, we get rid of the colons, the commas, and separate each keyword by a newline. They are useful exactly because they remove the verbosity when writing blocks of code. Most of the time, you will use the block syntax, but it is good to know they are equivalent. -This plays an important role in the language as it allows Elixir syntax to stay small but still expressive. We only need few data structures to represent the language, a topic we will come back to when talking about [optional syntax](optional-syntax.md) and go in-depth when discussing [meta-programming](../quote-and-unquote.md). +This plays an important role in the language as it allows Elixir syntax to stay small but still expressive. We only need few data structures to represent the language, a topic we will come back to when talking about [optional syntax](optional-syntax.md) and go in-depth when discussing [meta-programming](../meta-programming/quote-and-unquote.md). With this out of the way, let's talk about maps. diff --git a/lib/elixir/pages/mix-and-otp/distributed-tasks.md b/lib/elixir/pages/mix-and-otp/distributed-tasks.md index d2a274f3c5f..608fd3c2dd3 100644 --- a/lib/elixir/pages/mix-and-otp/distributed-tasks.md +++ b/lib/elixir/pages/mix-and-otp/distributed-tasks.md @@ -90,7 +90,7 @@ There are three better alternatives to `Node.spawn_link/2` that we could use in 2. We could have a server running on the other node and send requests to that node via the `GenServer` API. For example, you can call a server on a remote node by using `GenServer.call({name, node}, arg)` or passing the remote process PID as the first argument -3. We could use [tasks](`Task`), which we have learned about in [a previous chapter](../getting-started/mix-otp/task-and-gen-tcp.md), as they can be spawned on both local and remote nodes +3. We could use [tasks](`Task`), which we have learned about in [a previous chapter](task-and-gen-tcp.md), as they can be spawned on both local and remote nodes The options above have different properties. The GenServer would serialize your requests on a single server, while tasks are effectively running asynchronously on the remote node, with the only serialization point being the spawning done by the supervisor. diff --git a/lib/elixir/pages/mix-and-otp/genservers.md b/lib/elixir/pages/mix-and-otp/genservers.md index 74da0577c36..734c810b7c2 100644 --- a/lib/elixir/pages/mix-and-otp/genservers.md +++ b/lib/elixir/pages/mix-and-otp/genservers.md @@ -1,6 +1,6 @@ # Client-server communication with GenServer -In the [previous chapter](../agents.md), we used agents to represent our buckets. In the [introduction to mix](../introduction-to-mix.md), we specified we would like to name each bucket so we can do the following: +In the [previous chapter](agents.md), we used agents to represent our buckets. In the [introduction to mix](introduction-to-mix.md), we specified we would like to name each bucket so we can do the following: ```elixir CREATE shopping diff --git a/lib/elixir/pages/references/syntax-reference.md b/lib/elixir/pages/references/syntax-reference.md index 857fe110700..f9ec122012b 100644 --- a/lib/elixir/pages/references/syntax-reference.md +++ b/lib/elixir/pages/references/syntax-reference.md @@ -395,7 +395,7 @@ end All of the constructs above are part of Elixir's syntax and have their own representation as part of the Elixir AST. This section will discuss the remaining constructs that are alternative representations of the constructs above. In other words, the constructs below can be represented in more than one way in your Elixir code and retain AST equivalence. We call this "Optional Syntax". -For a lightweight introduction to Elixir's Optional Syntax, [see this document](optional-syntax.md). Below we continue with a more complete reference. +For a lightweight introduction to Elixir's Optional Syntax, [see this document](../getting-started/optional-syntax.md). Below we continue with a more complete reference. ### Integers in other bases and Unicode code points From 56b62b64cb6f5698b47213d445bf510402f34e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 13 Nov 2023 12:28:39 +0100 Subject: [PATCH 049/185] Consider start line in MismatchedDelimiterError --- lib/elixir/lib/exception.ex | 37 ++++- lib/elixir/src/elixir_errors.erl | 5 +- .../test/elixir/kernel/diagnostics_test.exs | 155 ++++++++++++++---- 3 files changed, 156 insertions(+), 41 deletions(-) diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index 68bb06bc6a4..3a78f6bad3f 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -954,6 +954,7 @@ defmodule MismatchedDelimiterError do :file, :line, :column, + :line_offset, :end_line, :end_column, :opening_delimiter, @@ -968,6 +969,7 @@ defmodule MismatchedDelimiterError do column: start_column, end_line: end_line, end_column: end_column, + line_offset: line_offset, description: description, opening_delimiter: opening_delimiter, closing_delimiter: _closing_delimiter, @@ -979,13 +981,24 @@ defmodule MismatchedDelimiterError do lines = String.split(snippet, "\n") expected_delimiter = :elixir_tokenizer.terminator(opening_delimiter) - snippet = format_snippet(start_pos, end_pos, description, file, lines, expected_delimiter) + snippet = + format_snippet( + start_pos, + end_pos, + line_offset, + description, + file, + lines, + expected_delimiter + ) + format_message(file, end_line, end_column, snippet) end defp format_snippet( {start_line, _start_column} = start_pos, {end_line, end_column} = end_pos, + line_offset, description, file, lines, @@ -998,12 +1011,21 @@ defmodule MismatchedDelimiterError do relevant_lines = if end_line - start_line < @max_lines_shown do - line_range(lines, start_pos, end_pos, padding, max_digits, expected_delimiter) + line_range( + lines, + start_pos, + end_pos, + line_offset, + padding, + max_digits, + expected_delimiter + ) else trimmed_inbetween_lines( lines, start_pos, end_pos, + line_offset, padding, max_digits, expected_delimiter @@ -1022,6 +1044,7 @@ defmodule MismatchedDelimiterError do defp format_snippet( {start_line, start_column}, {end_line, end_column}, + line_offset, description, file, lines, @@ -1032,7 +1055,7 @@ defmodule MismatchedDelimiterError do general_padding = max(2, max_digits) + 1 padding = n_spaces(general_padding) - line = Enum.fetch!(lines, end_line - 1) + line = Enum.fetch!(lines, end_line - 1 - line_offset) formatted_line = [line_padding(end_line, max_digits), to_string(end_line), " │ ", line] mismatched_closing_line = @@ -1080,14 +1103,15 @@ defmodule MismatchedDelimiterError do lines, {start_line, start_column}, {end_line, end_column}, + line_offset, padding, max_digits, expected_delimiter ) do start_padding = line_padding(start_line, max_digits) end_padding = line_padding(end_line, max_digits) - first_line = Enum.fetch!(lines, start_line - 1) - last_line = Enum.fetch!(lines, end_line - 1) + first_line = Enum.fetch!(lines, start_line - 1 - line_offset) + last_line = Enum.fetch!(lines, end_line - 1 - line_offset) """ #{start_padding}#{start_line} │ #{first_line} @@ -1102,6 +1126,7 @@ defmodule MismatchedDelimiterError do lines, {start_line, start_column}, {end_line, end_column}, + line_offset, padding, max_digits, expected_delimiter @@ -1110,7 +1135,7 @@ defmodule MismatchedDelimiterError do end_line = end_line - 1 lines - |> Enum.slice(start_line..end_line) + |> Enum.slice((start_line - line_offset)..(end_line - line_offset)) |> Enum.zip_with(start_line..end_line, fn line, line_number -> line_number = line_number + 1 start_line = start_line + 1 diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index e3212502da1..3e3504027e1 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -404,9 +404,10 @@ parse_erl_term(Term) -> Parsed. raise_mismatched_delimiter(Location, File, Input, Message) -> - {InputString, _, _} = Input, + {InputString, StartLine, _} = Input, InputBinary = elixir_utils:characters_to_binary(InputString), - raise('Elixir.MismatchedDelimiterError', Message, [{file, File}, {snippet, InputBinary} | Location]). + KV = [{file, File}, {line_offset, StartLine - 1}, {snippet, InputBinary} | Location], + raise('Elixir.MismatchedDelimiterError', Message, KV). raise_reserved(Location, File, Input, Keyword) -> raise_snippet(Location, File, Input, 'Elixir.SyntaxError', diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index a34dbb4e1d0..07b1f1a6033 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -53,6 +53,28 @@ defmodule Kernel.DiagnosticsTest do """ end + test "same line with offset" do + output = + capture_raise( + """ + [1, 2, 3, 4, 5, 6) + """, + MismatchedDelimiterError, + line: 3 + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:3:18: + error: unexpected token: ) + │ + 3 │ [1, 2, 3, 4, 5, 6) + │ │ └ mismatched closing delimiter (expected "]") + │ └ unclosed delimiter + │ + └─ nofile:3:18\ + """ + end + test "two-line span" do output = capture_raise( @@ -76,6 +98,30 @@ defmodule Kernel.DiagnosticsTest do """ end + test "two-line span with offset" do + output = + capture_raise( + """ + [a, b, c + d, f, g} + """, + MismatchedDelimiterError, + line: 3 + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:4:9: + error: unexpected token: } + │ + 3 │ [a, b, c + │ └ unclosed delimiter + 4 │ d, f, g} + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:4:9\ + """ + end + test "many-line span" do output = capture_raise( @@ -103,7 +149,9 @@ defmodule Kernel.DiagnosticsTest do │ └─ nofile:5:5\ """ + end + test "many-line span with offset" do output = capture_raise( """ @@ -111,20 +159,21 @@ defmodule Kernel.DiagnosticsTest do IO.inspect(2 + 2) + 2 ) """, - MismatchedDelimiterError + MismatchedDelimiterError, + line: 3 ) assert output == """ - ** (MismatchedDelimiterError) mismatched delimiter found on nofile:3:1: + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:5:1: error: unexpected token: ) │ - 1 │ fn always_forget_end -> + 3 │ fn always_forget_end -> │ └ unclosed delimiter - 2 │ IO.inspect(2 + 2) + 2 - 3 │ ) + 4 │ IO.inspect(2 + 2) + 2 + 5 │ ) │ └ mismatched closing delimiter (expected "end") │ - └─ nofile:3:1\ + └─ nofile:5:1\ """ end @@ -290,16 +339,6 @@ defmodule Kernel.DiagnosticsTest do describe "compile-time exceptions" do test "SyntaxError (snippet)" do - expected = """ - ** (SyntaxError) invalid syntax found on nofile:1:17: - error: syntax error before: '*' - │ - 1 │ [1, 2, 3, 4, 5, *] - │ ^ - │ - └─ nofile:1:17\ - """ - output = capture_raise( """ @@ -308,20 +347,39 @@ defmodule Kernel.DiagnosticsTest do SyntaxError ) - assert output == expected + assert output == """ + ** (SyntaxError) invalid syntax found on nofile:1:17: + error: syntax error before: '*' + │ + 1 │ [1, 2, 3, 4, 5, *] + │ ^ + │ + └─ nofile:1:17\ + """ end - test "TokenMissingError (snippet)" do - expected = """ - ** (TokenMissingError) token missing on nofile:1:4: - error: syntax error: expression is incomplete - │ - 1 │ 1 + - │ ^ - │ - └─ nofile:1:4\ - """ + test "SyntaxError (snippet) with offset" do + output = + capture_raise( + """ + [1, 2, 3, 4, 5, *] + """, + SyntaxError, + line: 3 + ) + + assert output == """ + ** (SyntaxError) invalid syntax found on nofile:3:17: + error: syntax error before: '*' + │ + 3 │ [1, 2, 3, 4, 5, *] + │ ^ + │ + └─ nofile:3:17\ + """ + end + test "TokenMissingError (snippet)" do output = capture_raise( """ @@ -330,7 +388,36 @@ defmodule Kernel.DiagnosticsTest do TokenMissingError ) - assert output == expected + assert output == """ + ** (TokenMissingError) token missing on nofile:1:4: + error: syntax error: expression is incomplete + │ + 1 │ 1 + + │ ^ + │ + └─ nofile:1:4\ + """ + end + + test "TokenMissingError (snippet) with offset" do + output = + capture_raise( + """ + 1 + + """, + TokenMissingError, + line: 3 + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:3:4: + error: syntax error: expression is incomplete + │ + 3 │ 1 + + │ ^ + │ + └─ nofile:3:4\ + """ end test "TokenMissingError (no snippet)" do @@ -419,7 +506,7 @@ defmodule Kernel.DiagnosticsTest do 1 - """, TokenMissingError, - fake_stacktrace + stacktrace: fake_stacktrace ) assert output == expected @@ -448,7 +535,7 @@ defmodule Kernel.DiagnosticsTest do 1 - """, TokenMissingError, - fake_stacktrace + stacktrace: fake_stacktrace ) assert output == expected @@ -1104,14 +1191,16 @@ defmodule Kernel.DiagnosticsTest do end) end - defp capture_raise(source, exception, mock_stacktrace \\ []) do + defp capture_raise(source, exception, opts \\ []) do + {stacktrace, opts} = Keyword.pop(opts, :stacktrace, []) + e = assert_raise exception, fn -> - ast = Code.string_to_quoted!(source, columns: true) + ast = Code.string_to_quoted!(source, [columns: true] ++ opts) Code.eval_quoted(ast) end - Exception.format(:error, e, mock_stacktrace) + Exception.format(:error, e, stacktrace) end defp purge(module) when is_atom(module) do From 901ec18aa7913031840538ed2e2b66c0c785367e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 14 Nov 2023 00:51:12 +0100 Subject: [PATCH 050/185] Handle error in Macro.to_string/1, closes #13102 --- lib/elixir/lib/code/normalizer.ex | 6 ++++-- lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index 03c8a706769..5a1c3c2fe68 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -349,13 +349,15 @@ defmodule Code.Normalizer do meta end + last = List.last(args) + cond do - Keyword.has_key?(meta, :do) or match?([{{:__block__, _, [:do]}, _} | _], List.last(args)) -> + Keyword.has_key?(meta, :do) or match?([{{:__block__, _, [:do]}, _} | _], last) -> # def foo do :ok end # def foo, do: :ok normalize_kw_blocks(form, meta, args, state) - match?([{:do, _} | _], List.last(args)) -> + match?([{:do, _} | _], last) and Keyword.keyword?(last) -> # Non normalized kw blocks line = state.parent_meta[:line] meta = meta ++ [do: [line: line], end: [line: line]] diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index 6cc2fce8438..f70cd98f525 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -229,6 +229,12 @@ defmodule Code.Normalizer.QuotedASTTest do test "invalid block" do assert quoted_to_string({:__block__, [], {:bar, [], []}}) == "{:__block__, [], {:bar, [], []}}" + + assert quoted_to_string({:foo, [], [{:do, :ok}, :not_keyword]}) == + "foo({:do, :ok}, :not_keyword)" + + assert quoted_to_string({:foo, [], [[{:do, :ok}, :not_keyword]]}) == + "foo([{:do, :ok}, :not_keyword])" end test "not in" do From aef86736666bf5d3f64d19a37d54301b58321e82 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Tue, 14 Nov 2023 09:58:36 +0100 Subject: [PATCH 051/185] Preserve column when translating typespecs (#13101) --- lib/elixir/lib/code/typespec.ex | 10 ++- lib/elixir/lib/kernel/typespec.ex | 92 +++++++++++++----------- lib/elixir/test/elixir/protocol_test.exs | 8 ++- lib/elixir/test/elixir/typespec_test.exs | 20 ++++-- 4 files changed, 79 insertions(+), 51 deletions(-) diff --git a/lib/elixir/lib/code/typespec.ex b/lib/elixir/lib/code/typespec.ex index cd38949bc70..017b2f05dcc 100644 --- a/lib/elixir/lib/code/typespec.ex +++ b/lib/elixir/lib/code/typespec.ex @@ -420,5 +420,13 @@ defmodule Code.Typespec do :error end - defp meta(anno), do: [line: :erl_anno.line(anno)] + defp meta(anno) do + case :erl_anno.location(anno) do + {line, column} -> + [line: line, column: column] + + line when is_integer(line) -> + [line: line] + end + end end diff --git a/lib/elixir/lib/kernel/typespec.ex b/lib/elixir/lib/kernel/typespec.ex index 2e5f88baf23..d95c34fc45d 100644 --- a/lib/elixir/lib/kernel/typespec.ex +++ b/lib/elixir/lib/kernel/typespec.ex @@ -385,17 +385,17 @@ defmodule Kernel.Typespec do compile_error(caller, error) end - line = line(meta) + location = location(meta) vars = Keyword.keys(guard) {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) {return, state} = typespec(return, vars, caller, state) - spec = {:type, line, :fun, [{:type, line, :product, args}, return]} + spec = {:type, location, :fun, [{:type, location, :product, args}, return]} {spec, state} = case guard_to_constraints(guard, vars, meta, caller, state) do {[], state} -> {spec, state} - {constraints, state} -> {{:type, line, :bounded_fun, [spec, constraints]}, state} + {constraints, state} -> {{:type, location, :bounded_fun, [spec, constraints]}, state} end ensure_no_unused_local_vars!(caller, state.local_vars) @@ -437,7 +437,7 @@ defmodule Kernel.Typespec do defp ensure_not_default(_), do: :ok defp guard_to_constraints(guard, vars, meta, caller, state) do - line = line(meta) + location = location(meta) fun = fn {_name, {:var, _, context}}, {constraints, state} when is_atom(context) -> @@ -445,9 +445,9 @@ defmodule Kernel.Typespec do {name, type}, {constraints, state} -> {spec, state} = typespec(type, vars, caller, state) - constraint = [{:atom, line, :is_subtype}, [{:var, line, name}, spec]] + constraint = [{:atom, location, :is_subtype}, [{:var, location, name}, spec]] state = update_local_vars(state, name) - {[{:type, line, :constraint, constraint} | constraints], state} + {[{:type, location, :constraint, constraint} | constraints], state} end {constraints, state} = :lists.foldl(fun, {[], state}, guard) @@ -456,21 +456,27 @@ defmodule Kernel.Typespec do ## To typespec conversion - defp line(meta) do - Keyword.get(meta, :line, 0) + defp location(meta) do + line = Keyword.get(meta, :line, 0) + + if column = Keyword.get(meta, :column) do + {line, column} + else + line + end end # Handle unions defp typespec({:|, meta, [_, _]} = exprs, vars, caller, state) do exprs = collect_union(exprs) {union, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, exprs) - {{:type, line(meta), :union, union}, state} + {{:type, location(meta), :union, union}, state} end # Handle binaries defp typespec({:<<>>, meta, []}, _, _, state) do - line = line(meta) - {{:type, line, :binary, [{:integer, line, 0}, {:integer, line, 0}]}, state} + location = location(meta) + {{:type, location, :binary, [{:integer, location, 0}, {:integer, location, 0}]}, state} end defp typespec( @@ -480,14 +486,18 @@ defmodule Kernel.Typespec do state ) when is_atom(ctx1) and is_atom(ctx2) and unit in 1..256 do - line = line(meta) - {{:type, line, :binary, [{:integer, line, 0}, {:integer, line(unit_meta), unit}]}, state} + location = location(meta) + + {{:type, location, :binary, [{:integer, location, 0}, {:integer, location(unit_meta), unit}]}, + state} end defp typespec({:<<>>, meta, [{:"::", size_meta, [{:_, _, ctx}, size]}]}, _, _, state) when is_atom(ctx) and is_integer(size) and size >= 0 do - line = line(meta) - {{:type, line, :binary, [{:integer, line(size_meta), size}, {:integer, line, 0}]}, state} + location = location(meta) + + {{:type, location, :binary, [{:integer, location(size_meta), size}, {:integer, location, 0}]}, + state} end defp typespec( @@ -505,8 +515,8 @@ defmodule Kernel.Typespec do ) when is_atom(ctx1) and is_atom(ctx2) and is_atom(ctx3) and is_integer(size) and size >= 0 and unit in 1..256 do - args = [{:integer, line(size_meta), size}, {:integer, line(unit_meta), unit}] - {{:type, line(meta), :binary, args}, state} + args = [{:integer, location(size_meta), size}, {:integer, location(unit_meta), unit}] + {{:type, location(meta), :binary, args}, state} end defp typespec({:<<>>, _meta, _args}, _vars, caller, _state) do @@ -519,7 +529,7 @@ defmodule Kernel.Typespec do ## Handle maps and structs defp typespec({:map, meta, args}, _vars, _caller, state) when args == [] or is_atom(args) do - {{:type, line(meta), :map, :any}, state} + {{:type, location(meta), :map, :any}, state} end defp typespec({:%{}, meta, fields} = map, vars, caller, state) do @@ -527,17 +537,17 @@ defmodule Kernel.Typespec do {{:required, meta2, [k]}, v}, state -> {arg1, state} = typespec(k, vars, caller, state) {arg2, state} = typespec(v, vars, caller, state) - {{:type, line(meta2), :map_field_exact, [arg1, arg2]}, state} + {{:type, location(meta2), :map_field_exact, [arg1, arg2]}, state} {{:optional, meta2, [k]}, v}, state -> {arg1, state} = typespec(k, vars, caller, state) {arg2, state} = typespec(v, vars, caller, state) - {{:type, line(meta2), :map_field_assoc, [arg1, arg2]}, state} + {{:type, location(meta2), :map_field_assoc, [arg1, arg2]}, state} {k, v}, state -> {arg1, state} = typespec(k, vars, caller, state) {arg2, state} = typespec(v, vars, caller, state) - {{:type, line(meta), :map_field_exact, [arg1, arg2]}, state} + {{:type, location(meta), :map_field_exact, [arg1, arg2]}, state} {:|, _, [_, _]}, _state -> error = @@ -551,7 +561,7 @@ defmodule Kernel.Typespec do end {fields, state} = :lists.mapfoldl(fun, state, fields) - {{:type, line(meta), :map, fields}, state} + {{:type, location(meta), :map, fields}, state} end defp typespec({:%, _, [name, {:%{}, meta, fields}]} = node, vars, caller, state) do @@ -644,7 +654,7 @@ defmodule Kernel.Typespec do {right, state} = typespec(right, vars, caller, state) :ok = validate_range(left, right, caller) - {{:type, line(meta), :range, [left, right]}, state} + {{:type, location(meta), :range, [left, right]}, state} end # Handle special forms @@ -668,7 +678,7 @@ defmodule Kernel.Typespec do pair -> pair end - {{:type, line(meta), :fun, fun_args}, state} + {{:type, location(meta), :fun, fun_args}, state} end # Handle type operator @@ -691,10 +701,10 @@ defmodule Kernel.Typespec do # This may be generating an invalid typespec but we need to generate it # to avoid breaking existing code that was valid but only broke Dialyzer {right, state} = typespec(expr, vars, caller, state) - {{:ann_type, line(meta), [{:var, line(var_meta), var_name}, right]}, state} + {{:ann_type, location(meta), [{:var, location(var_meta), var_name}, right]}, state} {right, state} -> - {{:ann_type, line(meta), [{:var, line(var_meta), var_name}, right]}, state} + {{:ann_type, location(meta), [{:var, location(var_meta), var_name}, right]}, state} end end @@ -723,13 +733,13 @@ defmodule Kernel.Typespec do {left, state} = typespec(left, vars, caller, state) state = %{state | undefined_type_error_enabled?: true} {right, state} = typespec(right, vars, caller, state) - {{:ann_type, line(meta), [left, right]}, state} + {{:ann_type, location(meta), [left, right]}, state} end # Handle unary ops defp typespec({op, meta, [integer]}, _, _, state) when op in [:+, :-] and is_integer(integer) do - line = line(meta) - {{:op, line, op, {:integer, line, integer}}, state} + location = location(meta) + {{:op, location, op, {:integer, location, integer}}, state} end # Handle remote calls in the form of @module_attribute.type. @@ -778,12 +788,12 @@ defmodule Kernel.Typespec do # Handle tuples defp typespec({:tuple, meta, []}, _vars, _caller, state) do - {{:type, line(meta), :tuple, :any}, state} + {{:type, location(meta), :tuple, :any}, state} end defp typespec({:{}, meta, t}, vars, caller, state) when is_list(t) do {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, t) - {{:type, line(meta), :tuple, args}, state} + {{:type, location(meta), :tuple, args}, state} end defp typespec({left, right}, vars, caller, state) do @@ -799,7 +809,7 @@ defmodule Kernel.Typespec do defp typespec({name, meta, atom}, vars, caller, state) when is_atom(atom) do if :lists.member(name, vars) do state = update_local_vars(state, name) - {{:var, line(meta), name}, state} + {{:var, location(meta), name}, state} else typespec({name, meta, []}, vars, caller, state) end @@ -814,7 +824,7 @@ defmodule Kernel.Typespec do IO.warn(warning, caller) {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) - {{:type, line(meta), :string, args}, state} + {{:type, location(meta), :string, args}, state} end defp typespec({:nonempty_string, meta, args}, vars, caller, state) do @@ -825,7 +835,7 @@ defmodule Kernel.Typespec do IO.warn(warning, caller) {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) - {{:type, line(meta), :nonempty_string, args}, state} + {{:type, location(meta), :nonempty_string, args}, state} end defp typespec({type, _meta, []}, vars, caller, state) when type in [:charlist, :char_list] do @@ -855,7 +865,7 @@ defmodule Kernel.Typespec do defp typespec({:fun, meta, args}, vars, caller, state) do {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) - {{:type, line(meta), :fun, args}, state} + {{:type, location(meta), :fun, args}, state} end defp typespec({:..., _meta, _args}, _vars, caller, _state) do @@ -872,7 +882,7 @@ defmodule Kernel.Typespec do case :erl_internal.is_type(name, arity) do true -> - {{:type, line(meta), name, args}, state} + {{:type, location(meta), name, args}, state} false -> if state.undefined_type_error_enabled? and @@ -890,7 +900,7 @@ defmodule Kernel.Typespec do %{state | used_type_pairs: [{name, arity} | state.used_type_pairs]} end - {{:user_type, line(meta), name, args}, state} + {{:user_type, location(meta), name, args}, state} end end @@ -963,7 +973,7 @@ defmodule Kernel.Typespec do defp remote_type({remote, meta, name, args}, vars, caller, state) do {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) - {{:remote_type, line(meta), [remote, name, args]}, state} + {{:remote_type, location(meta), [remote, name, args]}, state} end defp collect_union({:|, _, [a, b]}), do: [a | collect_union(b)] @@ -996,16 +1006,16 @@ defmodule Kernel.Typespec do end defp fn_args(meta, [{:..., _, _}], _vars, _caller, state) do - {{:type, line(meta), :any}, state} + {{:type, location(meta), :any}, state} end defp fn_args(meta, args, vars, caller, state) do {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) - {{:type, line(meta), :product, args}, state} + {{:type, location(meta), :product, args}, state} end defp variable({name, meta, args}) when is_atom(name) and is_atom(args) do - {:var, line(meta), name} + {:var, location(meta), name} end defp variable(expr), do: expr diff --git a/lib/elixir/test/elixir/protocol_test.exs b/lib/elixir/test/elixir/protocol_test.exs index e3754578a86..9931917bf11 100644 --- a/lib/elixir/test/elixir/protocol_test.exs +++ b/lib/elixir/test/elixir/protocol_test.exs @@ -154,8 +154,12 @@ defmodule ProtocolTest do end test "protocol defines callbacks" do - assert [{:type, 13, :fun, args}] = get_callbacks(@sample_binary, :ok, 1) - assert args == [{:type, 13, :product, [{:user_type, 13, :t, []}]}, {:type, 13, :boolean, []}] + assert [{:type, {13, 19}, :fun, args}] = get_callbacks(@sample_binary, :ok, 1) + + assert args == [ + {:type, {13, 19}, :product, [{:user_type, {13, 16}, :t, []}]}, + {:type, {13, 22}, :boolean, []} + ] assert [{:type, 23, :fun, args}] = get_callbacks(@with_any_binary, :ok, 1) assert args == [{:type, 23, :product, [{:user_type, 23, :t, []}]}, {:type, 23, :term, []}] diff --git a/lib/elixir/test/elixir/typespec_test.exs b/lib/elixir/test/elixir/typespec_test.exs index f08079b42a7..347670981ea 100644 --- a/lib/elixir/test/elixir/typespec_test.exs +++ b/lib/elixir/test/elixir/typespec_test.exs @@ -1582,23 +1582,29 @@ defmodule TypespecTest do """) [type: type] = types(:typespec_test_mod) - line = 5 assert Code.Typespec.type_to_quoted(type) == - {:"::", [], [{:t, [], [{:x, [line: line], nil}]}, [{:x, [line: line], nil}]]} + {:"::", [], + [ + {:t, [], [{:x, meta(5, 9), nil}]}, + [{:x, meta(5, 20), nil}] + ]} [{{:f, 1}, [spec]}] = specs(:typespec_test_mod) - line = 7 assert Code.Typespec.spec_to_quoted(:f, spec) == - {:when, [line: line], + {:when, meta(7, 8), [ - {:"::", [line: line], - [{:f, [line: line], [{:x, [line: line], nil}]}, {:x, [line: line], nil}]}, - [x: {:var, [line: line], nil}] + {:"::", meta(7, 8), + [{:f, meta(7, 8), [{:x, meta(7, 9), nil}]}, {:x, meta(7, 15), nil}]}, + [x: {:var, meta(7, 8), nil}] ]} end + defp meta(line, column) do + [line: line, column: column] + end + defp erlc(context, module, code) do dir = context.tmp_dir From 58b8b93ee698ba19a0935c357df3ed95f52f5b19 Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Wed, 15 Nov 2023 20:50:10 +0900 Subject: [PATCH 052/185] Auto infer size of matched variable in bitstrings (#13106) --- lib/elixir/lib/kernel.ex | 3 -- lib/elixir/src/elixir_bitstring.erl | 21 ++++++++--- lib/elixir/test/elixir/kernel/binary_test.exs | 36 +++++++++++-------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/lib/elixir/lib/kernel.ex b/lib/elixir/lib/kernel.ex index 7b37d181635..41859e34ff3 100644 --- a/lib/elixir/lib/kernel.ex +++ b/lib/elixir/lib/kernel.ex @@ -2064,9 +2064,6 @@ defmodule Kernel do {var, _, nil} when is_atom(var) -> invalid_concat_left_argument_error(Atom.to_string(var)) - {:^, _, [{var, _, nil}]} when is_atom(var) -> - invalid_concat_left_argument_error("^#{Atom.to_string(var)}") - _ -> expanded_arg end diff --git a/lib/elixir/src/elixir_bitstring.erl b/lib/elixir/src/elixir_bitstring.erl index 96d4cd92c4b..1d85535d069 100644 --- a/lib/elixir/src/elixir_bitstring.erl +++ b/lib/elixir/src/elixir_bitstring.erl @@ -35,7 +35,12 @@ expand(BitstrMeta, Fun, [{'::', Meta, [Left, Right]} | T], Acc, S, E, Alignment, MatchOrRequireSize = RequireSize or is_match_size(T, EL), EType = expr_type(ELeft), - {ERight, EAlignment, SS, ES} = expand_specs(EType, Meta, Right, SL, OriginalS, EL, MatchOrRequireSize), + ExpectSize = case ELeft of + {'^', _, [{_, _, _}]} -> {infer, ELeft}; + _ when MatchOrRequireSize -> required; + _ -> optional + end, + {ERight, EAlignment, SS, ES} = expand_specs(EType, Meta, Right, SL, OriginalS, EL, ExpectSize), EAcc = concat_or_prepend_bitstring(Meta, ELeft, ERight, Acc, ES, MatchOrRequireSize), expand(BitstrMeta, Fun, T, EAcc, {SS, OriginalS}, ES, alignment(Alignment, EAlignment), RequireSize); @@ -147,7 +152,7 @@ expand_expr(Meta, Component, Fun, S, E) -> %% Expands and normalizes types of a bitstring. -expand_specs(ExprType, Meta, Info, S, OriginalS, E, RequireSize) -> +expand_specs(ExprType, Meta, Info, S, OriginalS, E, ExpectSize) -> Default = #{size => default, unit => default, @@ -158,11 +163,17 @@ expand_specs(ExprType, Meta, Info, S, OriginalS, E, RequireSize) -> expand_each_spec(Meta, unpack_specs(Info, []), Default, S, OriginalS, E), MergedType = type(Meta, ExprType, Type, E), - validate_size_required(Meta, RequireSize, ExprType, MergedType, Size, ES), + validate_size_required(Meta, ExpectSize, ExprType, MergedType, Size, ES), SizeAndUnit = size_and_unit(Meta, ExprType, Size, Unit, ES), Alignment = compute_alignment(MergedType, Size, Unit), - [H | T] = build_spec(Meta, Size, Unit, MergedType, Endianness, Sign, SizeAndUnit, ES), + MaybeInferredSize = case {ExpectSize, MergedType, SizeAndUnit} of + {{infer, PinnedVar}, binary, []} -> [{size, Meta, [{{'.', Meta, [erlang, byte_size]}, Meta, [PinnedVar]}]}]; + {{infer, PinnedVar}, bitstring, []} -> [{size, Meta, [{{'.', Meta, [erlang, bit_size]}, Meta, [PinnedVar]}]}]; + _ -> SizeAndUnit + end, + + [H | T] = build_spec(Meta, Size, Unit, MergedType, Endianness, Sign, MaybeInferredSize, ES), {lists:foldl(fun(I, Acc) -> {'-', Meta, [Acc, I]} end, H, T), Alignment, SS, ES}. type(_, default, default, _) -> @@ -276,7 +287,7 @@ validate_spec_arg(Meta, unit, Value, _S, _OriginalS, E) when not is_integer(Valu validate_spec_arg(_Meta, _Key, _Value, _S, _OriginalS, _E) -> ok. -validate_size_required(Meta, true, default, Type, default, E) when Type == binary; Type == bitstring -> +validate_size_required(Meta, required, default, Type, default, E) when Type == binary; Type == bitstring -> function_error(Meta, E, ?MODULE, unsized_binary), ok; validate_size_required(_, _, _, _, _, _) -> diff --git a/lib/elixir/test/elixir/kernel/binary_test.exs b/lib/elixir/test/elixir/kernel/binary_test.exs index 7f2eb9a35e5..fd26ad3aee3 100644 --- a/lib/elixir/test/elixir/kernel/binary_test.exs +++ b/lib/elixir/test/elixir/kernel/binary_test.exs @@ -128,20 +128,6 @@ defmodule Kernel.BinaryTest do assert_raise ArgumentError, message, fn -> Code.eval_string(~s["a" <> b <> "c" = "abc"]) end - - assert_raise ArgumentError, message, fn -> - Code.eval_string(~s[ - a = "a" - ^a <> "b" = "ab" - ]) - end - - assert_raise ArgumentError, message, fn -> - Code.eval_string(~s[ - b = "b" - "a" <> ^b <> "c" = "abc" - ]) - end end test "hex" do @@ -269,6 +255,28 @@ defmodule Kernel.BinaryTest do assert <<1::size((^foo).bar)>> = <<1::5>> end + test "automatic size computation of matched bitsyntax variable" do + var = "foo" + <<^var::binary, rest::binary>> = "foobar" + assert rest == "bar" + + <<^var::bytes, rest::bytes>> = "foobar" + assert rest == "bar" + + ^var <> rest = "foobar" + assert rest == "bar" + + var = <<0, 1>> + <<^var::bitstring, rest::bitstring>> = <<0, 1, 2, 3>> + assert rest == <<2, 3>> + + <<^var::bits, rest::bits>> = <<0, 1, 2, 3>> + assert rest == <<2, 3>> + + ^var <> rest = <<0, 1, 2, 3>> + assert rest == <<2, 3>> + end + defmacro signed_16 do quote do big - signed - integer - unit(16) From 0a9ee79fbba5d307b0436ee4d54588be735ac8dd Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Wed, 15 Nov 2023 21:35:25 +0900 Subject: [PATCH 053/185] Formatter keeps quotes in atom keys (#13108) --- lib/elixir/lib/code/formatter.ex | 9 +++++---- .../test/elixir/code_formatter/containers_test.exs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex index eb1d0dd773e..5c84f675feb 100644 --- a/lib/elixir/lib/code/formatter.ex +++ b/lib/elixir/lib/code/formatter.ex @@ -520,14 +520,15 @@ defmodule Code.Formatter do {string(~S{"..//":}), state} {:__block__, _, [atom]} when is_atom(atom) -> - key = + iodata = if Macro.classify_atom(atom) in [:identifier, :unquoted] do - IO.iodata_to_binary([Atom.to_string(atom), ?:]) + [Atom.to_string(atom), ?:] else - IO.iodata_to_binary([?", Atom.to_string(atom), ?", ?:]) + [?", atom |> Atom.to_string() |> String.replace("\"", "\\\""), ?", ?:] end - {string(key) |> color(:atom, state.inspect_opts), state} + {iodata |> IO.iodata_to_binary() |> string() |> color(:atom, state.inspect_opts), + state} {{:., _, [:erlang, :binary_to_atom]}, _, [{:<<>>, _, entries}, :utf8]} -> interpolation_to_algebra(entries, @double_quote, state, "\"", "\":") diff --git a/lib/elixir/test/elixir/code_formatter/containers_test.exs b/lib/elixir/test/elixir/code_formatter/containers_test.exs index 9f3b3bc5d51..8e7b5e430d1 100644 --- a/lib/elixir/test/elixir/code_formatter/containers_test.exs +++ b/lib/elixir/test/elixir/code_formatter/containers_test.exs @@ -203,6 +203,7 @@ defmodule Code.Formatter.ContainersTest do assert_same ~S(["\w": 1, "\\w": 2]) assert_same ~S(["Elixir.Foo": 1, "Elixir.Bar": 2]) assert_format ~S(["Foo": 1, "Bar": 2]), ~S([Foo: 1, Bar: 2]) + assert_same ~S(["with \"scare quotes\"": 1]) end test "with operators keyword lists" do From 250b48aa3b60e587d9cd9d68cdd551149945913e Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 16 Nov 2023 13:59:21 +0100 Subject: [PATCH 054/185] Update `Mix.Config` mentions (#13115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ExDoc main emitted these warnings on Elixir main: ``` warning: documentation references module "Mix.Config" but it is hidden │ 49 │ `Mix.Config`, which was specific to Mix and has been deprecated. │ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ │ └─ lib/elixir/lib/config.ex:49: Config (module) warning: documentation references module "Mix.Config" but it is hidden │ 51 │ You can leverage `Config` instead of `Mix.Config` in three steps. The first │ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ │ └─ lib/elixir/lib/config.ex:51: Config (module) ``` --- lib/elixir/lib/config.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/config.ex b/lib/elixir/lib/config.ex index cc39e43baa5..abde56cb1d4 100644 --- a/lib/elixir/lib/config.ex +++ b/lib/elixir/lib/config.ex @@ -46,9 +46,9 @@ defmodule Config do ## Migrating from `use Mix.Config` The `Config` module in Elixir was introduced in v1.9 as a replacement to - `Mix.Config`, which was specific to Mix and has been deprecated. + `use Mix.Config`, which was specific to Mix and has been deprecated. - You can leverage `Config` instead of `Mix.Config` in three steps. The first + You can leverage `Config` instead of `use Mix.Config` in three steps. The first step is to replace `use Mix.Config` at the top of your config files by `import Config`. From 758da1e311527c79939502bfdc0551a781b37aa0 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 16 Nov 2023 14:00:20 +0100 Subject: [PATCH 055/185] Update Mix.Task.preferred_cli_env/1 docs (#13114) --- lib/mix/lib/mix/task.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/mix/lib/mix/task.ex b/lib/mix/lib/mix/task.ex index b5a4b81c5b7..8be509a0405 100644 --- a/lib/mix/lib/mix/task.ex +++ b/lib/mix/lib/mix/task.ex @@ -239,6 +239,9 @@ defmodule Mix.Task do Mix.ProjectStack.recursing() != nil end + @doc """ + Available for backwards compatibility. + """ @deprecated "Configure the environment in your mix.exs" defdelegate preferred_cli_env(task), to: Mix.CLI From c150876e5fb645ee37581642fbd8d1c109a3ee00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 22 Nov 2023 08:54:39 +0800 Subject: [PATCH 056/185] Remove warning on non-ambiguous nullary remote call --- lib/elixir/src/elixir_expand.erl | 18 ++++++------------ lib/elixir/test/elixir/kernel/warning_test.exs | 12 ------------ 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/lib/elixir/src/elixir_expand.erl b/lib/elixir/src/elixir_expand.erl index de71ccf3779..41dc113fb66 100644 --- a/lib/elixir/src/elixir_expand.erl +++ b/lib/elixir/src/elixir_expand.erl @@ -402,16 +402,12 @@ expand({Atom, Meta, Args}, S, E) when is_atom(Atom), is_list(Meta), is_list(Args %% Remote calls -expand({{'.', DotMeta, [Left, Right]}, Meta, Args} = Expr, S, E) +expand({{'.', DotMeta, [Left, Right]}, Meta, Args}, S, E) when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> {ELeft, SL, EL} = expand(Left, elixir_env:prepare_write(S), E), - NoParens = lists:keyfind(no_parens, 1, Meta), - - (is_atom(ELeft) and (Args =:= []) and (NoParens =:= {no_parens, true})) andalso - elixir_errors:file_warn(Meta, E, ?MODULE, {remote_nullary_no_parens, Expr}), elixir_dispatch:dispatch_require(Meta, ELeft, Right, Args, S, EL, fun(AR, AF, AA) -> - expand_remote(AR, DotMeta, AF, Meta, NoParens, AA, S, SL, EL) + expand_remote(AR, DotMeta, AF, Meta, AA, S, SL, EL) end); %% Anonymous calls @@ -858,12 +854,13 @@ expand_local(Meta, Name, Args, _S, #{function := nil} = E) -> %% Remote -expand_remote(Receiver, DotMeta, Right, Meta, NoParens, Args, S, SL, #{context := Context} = E) when is_atom(Receiver) or is_tuple(Receiver) -> +expand_remote(Receiver, DotMeta, Right, Meta, Args, S, SL, #{context := Context} = E) + when is_atom(Receiver) or is_tuple(Receiver) -> assert_no_clauses(Right, Meta, Args, E), if Context =:= guard, is_tuple(Receiver) -> - (NoParens /= {no_parens, true}) andalso + (lists:keyfind(no_parens, 1, Meta) /= {no_parens, true}) andalso function_error(Meta, E, ?MODULE, {parens_map_lookup, Receiver, Right, guard_context(S)}), {{{'.', DotMeta, [Receiver, Right]}, Meta, []}, SL, E}; @@ -881,7 +878,7 @@ expand_remote(Receiver, DotMeta, Right, Meta, NoParens, Args, S, SL, #{context : file_error(Meta, E, elixir_rewrite, Error) end end; -expand_remote(Receiver, DotMeta, Right, Meta, _NoParens, Args, _, _, E) -> +expand_remote(Receiver, DotMeta, Right, Meta, Args, _, _, E) -> Call = {{'.', DotMeta, [Receiver, Right]}, Meta, Args}, file_error(Meta, E, ?MODULE, {invalid_call, Call}). @@ -1171,9 +1168,6 @@ assert_no_underscore_clause_in_cond(_Other, _E) -> guard_context(#elixir_ex{prematch={_, _, {bitsize, _}}}) -> "bitstring size specifier"; guard_context(_) -> "guards". -format_error({remote_nullary_no_parens, Expr}) -> - String = 'Elixir.String':replace_suffix('Elixir.Macro':to_string(Expr), <<"()">>, <<>>), - io_lib:format("parentheses are required for function calls with no arguments, got: ~ts", [String]); format_error(invalid_match_on_zero_float) -> "pattern matching on 0.0 is equivalent to matching only on +0.0 from Erlang/OTP 27+. Instead you must match on +0.0 or -0.0"; format_error({useless_literal, Term}) -> diff --git a/lib/elixir/test/elixir/kernel/warning_test.exs b/lib/elixir/test/elixir/kernel/warning_test.exs index d91cf99a106..518da3c4c85 100644 --- a/lib/elixir/test/elixir/kernel/warning_test.exs +++ b/lib/elixir/test/elixir/kernel/warning_test.exs @@ -1175,18 +1175,6 @@ defmodule Kernel.WarningTest do purge(Sample) end - test "parens on nullary remote call" do - assert_warn_eval( - [ - "nofile:1:8", - "parentheses are required for function calls with no arguments, got: System.version" - ], - "System.version" - ) - after - purge(Sample) - end - test "parens with module attribute" do assert_warn_eval( [ From 1ff5e0c88bfa6d7c1ca47be6495d87f055db31f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 22 Nov 2023 09:23:01 +0800 Subject: [PATCH 057/185] Improve Logger docs, closes #13119 --- lib/logger/lib/logger.ex | 61 ++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/lib/logger/lib/logger.ex b/lib/logger/lib/logger.ex index e6e89683373..ba0df43ca08 100644 --- a/lib/logger/lib/logger.ex +++ b/lib/logger/lib/logger.ex @@ -13,16 +13,16 @@ defmodule Logger do * Supports both message-based and structural logging. + * Integrate with Erlang's [`:logger'](`:logger`) and + support custom filters and handlers. + * Formats and truncates messages on the client - to avoid clogging `Logger` backends. + to avoid clogging `Logger` handlers. * Alternates between sync and async modes to remain performant when required but also apply back-pressure when under stress. - * Support for custom filters and handlers as provided by - Erlang's `:logger`. - * Allows overriding the logging level for a specific module, application or process. @@ -65,7 +65,7 @@ defmodule Logger do For example, `:info` takes precedence over `:debug`. If your log level is set to `:info`, then all `:info`, `:notice` and above will - be passed to backends. If your log level is set to `:alert`, only + be passed to handlers. If your log level is set to `:alert`, only `:alert` and `:emergency` will be printed. ## Message @@ -126,8 +126,8 @@ defmodule Logger do * `:crash_reason` - a two-element tuple with the throw/error/exit reason as first argument and the stacktrace as second. A throw will always be `{:nocatch, term}`. An error is always an `Exception` struct. All other - entries are exits. The console backend ignores this metadata by default - but it can be useful to other backends, such as the ones that report + entries are exits. The default formatter ignores this metadata by default + but it can be useful to certain handlers, such as the ones that report errors to third-party services There are two special metadata keys, `:module` and `:function`, which @@ -275,8 +275,8 @@ defmodule Logger do Remember that if you want to purge log calls from a dependency, the dependency must be recompiled. - For example, to configure the `:backends` and purge all calls that happen - at compile time with level lower than `:info` in a `config/config.exs` file: + For example, to purge all calls that happen at compile time with level + lower than `:info` in a `config/config.exs` file: config :logger, compile_time_purge_matching: [ @@ -300,7 +300,7 @@ defmodule Logger do * `:level` - the logging level. Attempting to log any message with severity less than the configured level will simply - cause the message to be ignored. Keep in mind that each backend + cause the message to be ignored. Keep in mind that each handler may have its specific level, too. In addition to levels mentioned above it also supports 2 "meta-levels": @@ -397,7 +397,7 @@ defmodule Logger do Prior to Elixir v1.15, custom logging could be achieved with Logger backends. The main API for writing Logger backends have been moved to the [`:logger_backends`](https://github.com/elixir-lang/logger_backends) - project. However, the backends are still part of Elixir for backwards + project. However, the backends API are still part of Elixir for backwards compatibility. Important remarks: @@ -428,9 +428,12 @@ defmodule Logger do Backends, you can still set `backends: [Logger.Backends.Console]` and place the configuration under `config :logger, Logger.Backends.Console`. Although consider using the [`:logger_backends`](https://github.com/elixir-lang/logger_backends) - project in such case, as `Logger.Backends.Console` itself will be deprecated + project in such cases, as `Logger.Backends.Console` itself will be deprecated in future releases + * `Logger.Backends` only receive `:debug`, `:info`, `:warning`, and `:error` + messages. `:notice` maps to `:info`. `:warn` amps to `:warnings`. + All others map to `:error` """ @type level :: @@ -933,37 +936,21 @@ defmodule Logger do defp add_elixir_domain(metadata), do: Map.put(metadata, :domain, [:elixir]) - translations = %{ - emergency: :error, - alert: :error, - critical: :error, - notice: :info - } - for level <- @levels do report = [something: :reported, this: level] - - extra = - if translation = translations[level] do - """ - - - This is reported as \"#{translation}\" in Elixir's - logger backends for backwards compatibility reasons. - - """ - end + metadata = [user_id: 42, request_id: "xU32kFa"] + article = if level in [:info, :error, :alert, :emergency], do: "an", else: "a" @doc """ - Logs a #{level} message. + Logs #{article} #{level} message. - Returns `:ok`.#{extra} + Returns `:ok`. ## Examples Logging a message (string or iodata): - Logger.#{level}("this is a #{level} message") + Logger.#{level}("this is #{article} #{level} message") Report message (maps or keywords): @@ -973,6 +960,14 @@ defmodule Logger do # as map Logger.#{level}(#{inspect(Map.new(report))}) + Report message with metadata (maps or keywords): + + # as a keyword list + Logger.#{level}("this is #{article} #{level} message", #{inspect(metadata)}) + + # as map + Logger.#{level}("this is #{article} #{level} message", #{inspect(Map.new(metadata))}) + """ # Only macros generated for the "new" Erlang levels are available since 1.11.0. Other From 2caacae2e43914725d36f5596d449d1347f2cf4b Mon Sep 17 00:00:00 2001 From: Andrea Leopardi Date: Wed, 22 Nov 2023 10:52:33 +0100 Subject: [PATCH 058/185] Add some specs and types to ExUnit.Formatter (#13130) --- lib/ex_unit/lib/ex_unit/formatter.ex | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib/ex_unit/lib/ex_unit/formatter.ex b/lib/ex_unit/lib/ex_unit/formatter.ex index ed639cea4b4..b3e73fd460d 100644 --- a/lib/ex_unit/lib/ex_unit/formatter.ex +++ b/lib/ex_unit/lib/ex_unit/formatter.ex @@ -67,6 +67,20 @@ defmodule ExUnit.Formatter do load: pos_integer | nil } + @typedoc """ + A function that this module calls to format various things. + """ + @typedoc since: "1.16.0" + @type formatter_callback :: (atom, term -> term) + + @typedoc """ + Width for formatting. + + For example, see `format_assertion_diff/4`. + """ + @typedoc since: "1.16.0" + @type width :: non_neg_integer | :infinity + import Exception, only: [format_stacktrace_entry: 1, format_file_line: 3] alias ExUnit.Diff @@ -155,6 +169,14 @@ defmodule ExUnit.Formatter do @doc """ Receives a test and formats its failure. """ + @spec format_test_failure( + ExUnit.Test.t(), + [failure], + non_neg_integer, + width, + formatter_callback + ) :: String.t() + when failure: {atom, term, Exception.stacktrace()} def format_test_failure(test, failures, counter, width, formatter) do %ExUnit.Test{name: name, module: module, tags: tags} = test @@ -177,6 +199,14 @@ defmodule ExUnit.Formatter do @doc """ Receives a test module and formats its failure. """ + @spec format_test_all_failure( + ExUnit.TestModule.t(), + [failure], + non_neg_integer, + width, + formatter_callback + ) :: String.t() + when failure: {atom, term, Exception.stacktrace()} def format_test_all_failure(test_module, failures, counter, width, formatter) do name = test_module.name @@ -280,6 +310,12 @@ defmodule ExUnit.Formatter do for formatted content, the width (may be `:infinity`), and the formatter callback function. """ + @spec format_assertion_diff( + %ExUnit.AssertionError{}, + non_neg_integer, + width, + formatter_callback + ) :: keyword def format_assertion_diff(assert_error, padding_size, width, formatter) def format_assertion_diff(%ExUnit.AssertionError{context: {:mailbox, _pins, []}}, _, _, _) do From 46a5c54844415d8b1b617b87ce76f2fa6e597842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Samson?= Date: Wed, 22 Nov 2023 17:45:54 +0100 Subject: [PATCH 059/185] Properly escape `\` in Path.wildcard docs (#13137) --- lib/elixir/lib/path.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/elixir/lib/path.ex b/lib/elixir/lib/path.ex index 0f18487a101..bba4de5ad70 100644 --- a/lib/elixir/lib/path.ex +++ b/lib/elixir/lib/path.ex @@ -759,9 +759,9 @@ defmodule Path do You may call `Path.expand/1` to normalize the path before invoking this function. - A character preceded by \ loses its special meaning. - Note that \ must be written as \\ in a string literal. - For example, "\\?*" will match any filename starting with ?. + A character preceded by `\\` loses its special meaning. + Note that `\\` must be written as `\\\\` in a string literal. + For example, `"\\\\?*"` will match any filename starting with `?.`. By default, the patterns `*` and `?` do not match files starting with a dot `.`. See the `:match_dot` option in the "Options" section From 2364f0499178a578f98e5d78d00d52c6c88d5db3 Mon Sep 17 00:00:00 2001 From: Andrea Leopardi Date: Thu, 23 Nov 2023 04:23:27 +0100 Subject: [PATCH 060/185] Add callback docs to ExUnit.Formatter (#13135) --- lib/ex_unit/lib/ex_unit/formatter.ex | 117 +++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 5 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/formatter.ex b/lib/ex_unit/lib/ex_unit/formatter.ex index b3e73fd460d..ab25f21d188 100644 --- a/lib/ex_unit/lib/ex_unit/formatter.ex +++ b/lib/ex_unit/lib/ex_unit/formatter.ex @@ -67,11 +67,88 @@ defmodule ExUnit.Formatter do load: pos_integer | nil } + @typedoc """ + Key passed to a formatter callback to format a diff. + + See `t:formatter_callback/0`. + """ + @typedoc since: "1.16.0" + @type formatter_callback_diff_key :: + :diff_delete + | :diff_delete_whitespace + | :diff_insert + | :diff_insert_whitespace + + @typedoc """ + Key passed to a formatter callback to format information. + + See `t:formatter_callback/0`. + """ + @typedoc since: "1.16.0" + @type formatter_callback_info_key :: + :extra_info + | :error_info + | :test_module_info + | :test_info + | :location_info + | :stacktrace_info + | :blame_diff + @typedoc """ A function that this module calls to format various things. + + You can pass this functions to various functions in this module, and use it + to customize the formatting of the output. For example, ExUnit's CLI formatter + uses this callback to colorize output. + + ## Keys + + The possible keys are: + + * `:diff_enabled?` - whether diffing is enabled. It receives a boolean + indicating whether diffing is enabled by default and returns a boolean + indicating whether diffing should be enabled for the current test. + + * `:diff_delete` and `:diff_delete_whitespace` - Should format a diff deletion, + with or without whitespace respectively. + + * `:diff_insert` and `:diff_insert_whitespace` - Should format a diff insertion, + with or without whitespace respectively. + + * `:extra_info` - Should format extra information, such as the `"code: "` label + that precedes code to show. + + * `:error_info` - Should format error information. + + * `:error_info` - Should format error information. + + * `:test_module_info` - Should format test module information. The message returned + when this key is passed precedes messages such as `"failure on setup_all callback [...]"`. + + * `:test_info` - Should format test information. + + * `:location_info` - Should format test location information. + + * `:stacktrace_info` - Should format stacktrace information. + + * `:blame_diff` - Should format a string of code. + + ## Examples + + For example, to format errors as *red strings* and everything else as is, you could define + a formatter callback function like this: + + formatter_callback = fn + :error_info, msg -> [:red, msg, :reset] |> IO.ANSI.format() |> IO.iodata_to_binary() + _key, value -> value + end + """ @typedoc since: "1.16.0" - @type formatter_callback :: (atom, term -> term) + @type formatter_callback :: + (:diff_enabled?, boolean -> boolean) + | (formatter_callback_diff_key, Inspect.Algebra.t() -> Inspect.Algebra.t()) + | (formatter_callback_info_key, String.t() -> String.t()) @typedoc """ Width for formatting. @@ -166,11 +243,20 @@ defmodule ExUnit.Formatter do end end - @doc """ - Receives a test and formats its failure. + @doc ~S""" + Receives a test and formats its failures. + + ## Examples + + iex> failure = {:error, catch_error(raise "oops"), _stacktrace = []} + iex> formatter_cb = fn _key, value -> value end + iex> test = %ExUnit.Test{name: :"it works", module: MyTest, tags: %{file: "file.ex", line: 7}} + iex> format_test_failure(test, [failure], 1, 80, formatter_cb) + " 1) it works (MyTest)\n file.ex:7\n ** (RuntimeError) oops\n" + """ @spec format_test_failure( - ExUnit.Test.t(), + test, [failure], non_neg_integer, width, @@ -196,8 +282,17 @@ defmodule ExUnit.Formatter do format_test_all_failure(test_case, failures, counter, width, formatter) end - @doc """ + @doc ~S""" Receives a test module and formats its failure. + + ## Examples + + iex> failure = {:error, catch_error(raise "oops"), _stacktrace = []} + iex> formatter_cb = fn _key, value -> value end + iex> test_module = %ExUnit.TestModule{name: Hello} + iex> format_test_all_failure(test_module, [failure], 1, 80, formatter_cb) + " 1) Hello: failure on setup_all callback, all tests have been invalidated\n ** (RuntimeError) oops\n" + """ @spec format_test_all_failure( ExUnit.TestModule.t(), @@ -309,6 +404,18 @@ defmodule ExUnit.Formatter do It expects the assertion error, the `padding_size` for formatted content, the width (may be `:infinity`), and the formatter callback function. + + ## Examples + + iex> error = assert_raise ExUnit.AssertionError, fn -> assert [1, 2] == [1, 3] end + iex> formatter_cb = fn + ...> :diff_enabled?, _ -> true + ...> _key, value -> value + ...> end + iex> keyword = format_assertion_diff(error, 5, 80, formatter_cb) + iex> for {key, val} <- keyword, do: {key, IO.iodata_to_binary(val)} + [left: "[1, 2]", right: "[1, 3]"] + """ @spec format_assertion_diff( %ExUnit.AssertionError{}, From adff7f6d346160615b629d61e319c991f0f95cbf Mon Sep 17 00:00:00 2001 From: Andrea Leopardi Date: Thu, 23 Nov 2023 04:23:54 +0100 Subject: [PATCH 061/185] Add t/0 types for some ExUnit exceptions (#13134) --- lib/ex_unit/lib/ex_unit/assertions.ex | 19 +++++++++++++++++++ lib/ex_unit/lib/ex_unit/formatter.ex | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/ex_unit/lib/ex_unit/assertions.ex b/lib/ex_unit/lib/ex_unit/assertions.ex index 9b4bd54d4ea..6ba920372e5 100644 --- a/lib/ex_unit/lib/ex_unit/assertions.ex +++ b/lib/ex_unit/lib/ex_unit/assertions.ex @@ -1,10 +1,23 @@ defmodule ExUnit.AssertionError do @moduledoc """ Raised to signal an assertion error. + + This is used by macros such as `ExUnit.Assertions.assert/1`. """ @no_value :ex_unit_no_meaningful_value + @typedoc since: "1.16.0" + @type t :: %__MODULE__{ + left: any, + right: any, + message: any, + expr: any, + args: any, + doctest: any, + context: any + } + defexception left: @no_value, right: @no_value, message: @no_value, @@ -16,6 +29,7 @@ defmodule ExUnit.AssertionError do @doc """ Indicates no meaningful value for a field. """ + @spec no_value :: atom def no_value do @no_value end @@ -31,6 +45,11 @@ defmodule ExUnit.MultiError do Raised to signal multiple errors happened in a test case. """ + @typedoc since: "1.16.0" + @type t :: %__MODULE__{ + errors: [{Exception.kind(), any, Exception.stacktrace()}] + } + defexception errors: [] @impl true diff --git a/lib/ex_unit/lib/ex_unit/formatter.ex b/lib/ex_unit/lib/ex_unit/formatter.ex index ab25f21d188..da41f6c3ef5 100644 --- a/lib/ex_unit/lib/ex_unit/formatter.ex +++ b/lib/ex_unit/lib/ex_unit/formatter.ex @@ -418,7 +418,7 @@ defmodule ExUnit.Formatter do """ @spec format_assertion_diff( - %ExUnit.AssertionError{}, + ExUnit.AssertionError.t(), non_neg_integer, width, formatter_callback From abd4f54d7872b2ec0ef5c4b584da70f08c3b7f74 Mon Sep 17 00:00:00 2001 From: Andrea Leopardi Date: Thu, 23 Nov 2023 12:08:39 +0100 Subject: [PATCH 062/185] Fix typo in Logger docs --- lib/logger/lib/logger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logger/lib/logger.ex b/lib/logger/lib/logger.ex index ba0df43ca08..bb10e5d063e 100644 --- a/lib/logger/lib/logger.ex +++ b/lib/logger/lib/logger.ex @@ -13,7 +13,7 @@ defmodule Logger do * Supports both message-based and structural logging. - * Integrate with Erlang's [`:logger'](`:logger`) and + * Integrate with Erlang's [`:logger`](`:logger`) and support custom filters and handlers. * Formats and truncates messages on the client From 6145599638115b958f2631a695acf7ffeaf0d969 Mon Sep 17 00:00:00 2001 From: Andrea Leopardi Date: Thu, 23 Nov 2023 12:16:15 +0100 Subject: [PATCH 063/185] Add t/0 types to remaining ExUnit exceptions (#13139) --- lib/ex_unit/lib/ex_unit.ex | 10 ++++++++++ lib/ex_unit/lib/ex_unit/case.ex | 15 +++++++++++++++ lib/ex_unit/lib/ex_unit/doc_test.ex | 6 ++++++ 3 files changed, 31 insertions(+) diff --git a/lib/ex_unit/lib/ex_unit.ex b/lib/ex_unit/lib/ex_unit.ex index 7a0ad68d7d3..a838ae58183 100644 --- a/lib/ex_unit/lib/ex_unit.ex +++ b/lib/ex_unit/lib/ex_unit.ex @@ -148,6 +148,16 @@ defmodule ExUnit do end defmodule TimeoutError do + @moduledoc """ + Exception raised when a test times out. + """ + + @typedoc since: "1.16.0" + @type t :: %__MODULE__{ + timeout: non_neg_integer, + type: String.t() + } + defexception [:timeout, :type] @impl true diff --git a/lib/ex_unit/lib/ex_unit/case.ex b/lib/ex_unit/lib/ex_unit/case.ex index 7cbeefa8281..7aebfd5b438 100644 --- a/lib/ex_unit/lib/ex_unit/case.ex +++ b/lib/ex_unit/lib/ex_unit/case.ex @@ -1,8 +1,23 @@ defmodule ExUnit.DuplicateTestError do + @moduledoc """ + Exception raised to indicate two or more tests with the same name. + """ + + @typedoc since: "1.16.0" + @type t :: %__MODULE__{message: String.t()} + defexception [:message] end defmodule ExUnit.DuplicateDescribeError do + @moduledoc """ + Exception raised to indicate two or more `describe` blocks with + the same name. + """ + + @typedoc since: "1.16.0" + @type t :: %__MODULE__{message: String.t()} + defexception [:message] end diff --git a/lib/ex_unit/lib/ex_unit/doc_test.ex b/lib/ex_unit/lib/ex_unit/doc_test.ex index 8224d4fe102..7d99db93399 100644 --- a/lib/ex_unit/lib/ex_unit/doc_test.ex +++ b/lib/ex_unit/lib/ex_unit/doc_test.ex @@ -152,6 +152,12 @@ defmodule ExUnit.DocTest do @opaque_type_regex ~r/#[\w\.]+ Date: Thu, 23 Nov 2023 08:08:31 -0600 Subject: [PATCH 064/185] Add Logger.levels/0 (#13136) Co-authored-by: c4710n --- lib/logger/lib/logger.ex | 7 +++++++ lib/logger/test/logger_test.exs | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/lib/logger/lib/logger.ex b/lib/logger/lib/logger.ex index bb10e5d063e..7eab7eafe95 100644 --- a/lib/logger/lib/logger.ex +++ b/lib/logger/lib/logger.ex @@ -445,6 +445,13 @@ defmodule Logger do @levels [:error, :info, :debug] ++ @new_erlang_levels @metadata :logger_level + @doc ~S""" + Returns all the available levels. + """ + @doc since: "1.16.0" + @spec levels() :: [level(), ...] + def levels(), do: @levels + @doc ~S""" Returns the default formatter used by Logger. diff --git a/lib/logger/test/logger_test.exs b/lib/logger/test/logger_test.exs index d9ee5eb8381..2d26722711d 100644 --- a/lib/logger/test/logger_test.exs +++ b/lib/logger/test/logger_test.exs @@ -16,6 +16,11 @@ defmodule LoggerTest do msg("module=LoggerTest #{text}") end + test "levels/0" do + assert [_ | _] = Logger.levels() + assert :info in Logger.levels() + end + test "level/0" do assert Logger.level() == :debug From aa0dcb9a704e44bb04014cece1d04bfc0e1283bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 23 Nov 2023 22:33:23 +0800 Subject: [PATCH 065/185] Fix prying functions with only literals, closes #13133 --- lib/iex/lib/iex/pry.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/iex/lib/iex/pry.ex b/lib/iex/lib/iex/pry.ex index 82842605f7b..32838c03971 100644 --- a/lib/iex/lib/iex/pry.ex +++ b/lib/iex/lib/iex/pry.ex @@ -519,7 +519,7 @@ defmodule IEx.Pry do next_binding = binding(expr, binding) {min_line, max_line} = line_range(expr, line) - if force? or (min_line > line and min_line != :infinity) do + if force? or min_line > line do pry_var = next_var(version) pry_binding = Map.to_list(binding) pry_opts = [line: min_line] ++ opts @@ -545,7 +545,9 @@ defmodule IEx.Pry do end defp line_range(ast, line) do - {_, min_max} = + # We want min_line to start from infinity because + # if it starts from line it will always just return line. + {_, {min, max}} = Macro.prewalk(ast, {:infinity, line}, fn {_, meta, _} = ast, {min_line, max_line} when is_list(meta) -> line = meta[:line] @@ -560,7 +562,7 @@ defmodule IEx.Pry do {ast, acc} end) - min_max + if min == :infinity, do: {line, max}, else: {min, max} end defp binding(ast, binding) do From d716bc2703c0ee3647c9f988d68ddb6100080022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 25 Nov 2023 10:34:38 +0800 Subject: [PATCH 066/185] Include both priv and include in releases, closes #13145 --- lib/mix/lib/mix/release.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/lib/mix/release.ex b/lib/mix/lib/mix/release.ex index be395713e69..ce3060a51f7 100644 --- a/lib/mix/lib/mix/release.ex +++ b/lib/mix/lib/mix/release.ex @@ -68,7 +68,7 @@ defmodule Mix.Release do @safe_modes [:permanent, :temporary, :transient] @unsafe_modes [:load, :none] @additional_chunks ~w(Attr)c - @copy_app_dirs ["priv"] + @copy_app_dirs ["priv", "include"] @doc false @spec from_config!(atom, keyword, keyword) :: t From 106539b5d23cce2923386a1a704edbdc78af2d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20M=C3=BCller?= Date: Mon, 4 Dec 2023 08:46:15 -0300 Subject: [PATCH 067/185] Improve unclosed delimiter messages (#13123) --- lib/elixir/lib/exception.ex | 482 ++++++++++-------- lib/elixir/src/elixir_errors.erl | 10 +- lib/elixir/src/elixir_tokenizer.erl | 14 +- .../test/elixir/kernel/diagnostics_test.exs | 147 +++++- lib/elixir/test/elixir/kernel/errors_test.exs | 8 +- lib/elixir/test/elixir/kernel/parser_test.exs | 4 +- lib/ex_unit/test/ex_unit/doc_test_test.exs | 13 +- 7 files changed, 447 insertions(+), 231 deletions(-) diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index 3a78f6bad3f..d6ff0ba80f3 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -793,6 +793,212 @@ defmodule Exception do col -> format_file_line_column(Keyword.get(opts, :file), Keyword.get(opts, :line), col, " ") end end + + @doc false + def format_snippet( + {start_line, _start_column} = start_pos, + {end_line, end_column} = end_pos, + line_offset, + description, + file, + lines, + start_message, + end_message + ) + when start_line < end_line do + max_digits = digits(end_line) + general_padding = max(2, max_digits) + 1 + padding = n_spaces(general_padding) + + relevant_lines = + if end_line - start_line < 5 do + line_range( + lines, + start_pos, + end_pos, + line_offset, + padding, + max_digits, + start_message, + end_message + ) + else + trimmed_inbetween_lines( + lines, + start_pos, + end_pos, + line_offset, + padding, + max_digits, + start_message, + end_message + ) + end + + """ + #{padding}#{red("error:")} #{pad_message(description, padding)} + #{padding}│ + #{relevant_lines} + #{padding}│ + #{padding}└─ #{Path.relative_to_cwd(file)}:#{end_line}:#{end_column}\ + """ + end + + def format_snippet( + {start_line, start_column}, + {end_line, end_column}, + line_offset, + description, + file, + lines, + start_message, + end_message + ) + when start_line == end_line do + max_digits = digits(end_line) + general_padding = max(2, max_digits) + 1 + padding = n_spaces(general_padding) + + line = Enum.fetch!(lines, end_line - 1 - line_offset) + formatted_line = [line_padding(end_line, max_digits), to_string(end_line), " │ ", line] + + mismatched_closing_line = + [ + n_spaces(start_column - 1), + red("│"), + format_end_message(end_column - start_column, end_message) + ] + + unclosed_delimiter_line = + [padding, " │ ", format_start_message(start_column, start_message)] + + below_line = [padding, " │ ", mismatched_closing_line, "\n", unclosed_delimiter_line] + + """ + #{padding}#{red("error:")} #{pad_message(description, padding)} + #{padding}│ + #{formatted_line} + #{below_line} + #{padding}│ + #{padding}└─ #{Path.relative_to_cwd(file)}:#{end_line}:#{end_column}\ + """ + end + + defp line_padding(line_number, max_digits) do + line_digits = digits(line_number) + + spacing = + if line_digits == 1 do + max(2, max_digits) + else + max_digits - line_digits + 1 + end + + n_spaces(spacing) + end + + defp n_spaces(n), do: String.duplicate(" ", n) + + defp digits(number, acc \\ 1) + defp digits(number, acc) when number < 10, do: acc + defp digits(number, acc), do: digits(div(number, 10), acc + 1) + + defp trimmed_inbetween_lines( + lines, + {start_line, start_column}, + {end_line, end_column}, + line_offset, + padding, + max_digits, + start_message, + end_message + ) do + start_padding = line_padding(start_line, max_digits) + end_padding = line_padding(end_line, max_digits) + first_line = Enum.fetch!(lines, start_line - 1 - line_offset) + last_line = Enum.fetch!(lines, end_line - 1 - line_offset) + + """ + #{start_padding}#{start_line} │ #{first_line} + #{padding}│ #{format_start_message(start_column, start_message)} + ... + #{end_padding}#{end_line} │ #{last_line} + #{padding}│ #{format_end_message(end_column, end_message)}\ + """ + end + + defp line_range( + lines, + {start_line, start_column}, + {end_line, end_column}, + line_offset, + padding, + max_digits, + start_message, + end_message + ) do + start_line = start_line - 1 + end_line = end_line - 1 + + lines + |> Enum.slice((start_line - line_offset)..(end_line - line_offset)) + |> Enum.zip_with(start_line..end_line, fn line, line_number -> + line_number = line_number + 1 + start_line = start_line + 1 + end_line = end_line + 1 + + line_padding = line_padding(line_number, max_digits) + + cond do + line_number == start_line -> + [ + line_padding, + to_string(line_number), + " │ ", + line, + "\n", + padding, + " │ ", + format_start_message(start_column, start_message) + ] + + line_number == end_line -> + [ + line_padding, + to_string(line_number), + " │ ", + line, + "\n", + padding, + " │ ", + format_end_message(end_column, end_message) + ] + + true -> + [line_padding, to_string(line_number), " │ ", line] + end + end) + |> Enum.intersperse("\n") + end + + defp format_end_message(end_column, message), + do: [ + n_spaces(end_column - 1), + red(message) + ] + + defp format_start_message(start_column, message), + do: [n_spaces(start_column - 1), red(message)] + + defp pad_message(message, padding), do: String.replace(message, "\n", "\n #{padding}") + + defp red(string) do + if IO.ANSI.enabled?() do + [IO.ANSI.red(), string, IO.ANSI.reset()] + else + string + end + end end # Some exceptions implement "message/1" instead of "exception/1" mostly @@ -948,8 +1154,6 @@ defmodule MismatchedDelimiterError do - `fn a -> )` """ - @max_lines_shown 5 - defexception [ :file, :line, @@ -981,219 +1185,24 @@ defmodule MismatchedDelimiterError do lines = String.split(snippet, "\n") expected_delimiter = :elixir_tokenizer.terminator(opening_delimiter) + start_message = "└ unclosed delimiter" + end_message = ~s/└ mismatched closing delimiter (expected "#{expected_delimiter}")/ + snippet = - format_snippet( + Exception.format_snippet( start_pos, end_pos, line_offset, description, file, lines, - expected_delimiter + start_message, + end_message ) format_message(file, end_line, end_column, snippet) end - defp format_snippet( - {start_line, _start_column} = start_pos, - {end_line, end_column} = end_pos, - line_offset, - description, - file, - lines, - expected_delimiter - ) - when start_line < end_line do - max_digits = digits(end_line) - general_padding = max(2, max_digits) + 1 - padding = n_spaces(general_padding) - - relevant_lines = - if end_line - start_line < @max_lines_shown do - line_range( - lines, - start_pos, - end_pos, - line_offset, - padding, - max_digits, - expected_delimiter - ) - else - trimmed_inbetween_lines( - lines, - start_pos, - end_pos, - line_offset, - padding, - max_digits, - expected_delimiter - ) - end - - """ - #{padding}#{red("error:")} #{pad_message(description, padding)} - #{padding}│ - #{relevant_lines} - #{padding}│ - #{padding}└─ #{Path.relative_to_cwd(file)}:#{end_line}:#{end_column}\ - """ - end - - defp format_snippet( - {start_line, start_column}, - {end_line, end_column}, - line_offset, - description, - file, - lines, - expected_delimiter - ) - when start_line == end_line do - max_digits = digits(end_line) - general_padding = max(2, max_digits) + 1 - padding = n_spaces(general_padding) - - line = Enum.fetch!(lines, end_line - 1 - line_offset) - formatted_line = [line_padding(end_line, max_digits), to_string(end_line), " │ ", line] - - mismatched_closing_line = - [ - n_spaces(start_column - 1), - red("│"), - mismatched_closing_delimiter(end_column - start_column, expected_delimiter) - ] - - unclosed_delimiter_line = - [padding, " │ ", unclosed_delimiter(start_column)] - - below_line = [padding, " │ ", mismatched_closing_line, "\n", unclosed_delimiter_line] - - """ - #{padding}#{red("error:")} #{pad_message(description, padding)} - #{padding}│ - #{formatted_line} - #{below_line} - #{padding}│ - #{padding}└─ #{Path.relative_to_cwd(file)}:#{end_line}:#{end_column}\ - """ - end - - defp line_padding(line_number, max_digits) do - line_digits = digits(line_number) - - spacing = - if line_digits == 1 do - max(2, max_digits) - else - max_digits - line_digits + 1 - end - - n_spaces(spacing) - end - - defp n_spaces(n), do: String.duplicate(" ", n) - - defp digits(number, acc \\ 1) - defp digits(number, acc) when number < 10, do: acc - defp digits(number, acc), do: digits(div(number, 10), acc + 1) - - defp trimmed_inbetween_lines( - lines, - {start_line, start_column}, - {end_line, end_column}, - line_offset, - padding, - max_digits, - expected_delimiter - ) do - start_padding = line_padding(start_line, max_digits) - end_padding = line_padding(end_line, max_digits) - first_line = Enum.fetch!(lines, start_line - 1 - line_offset) - last_line = Enum.fetch!(lines, end_line - 1 - line_offset) - - """ - #{start_padding}#{start_line} │ #{first_line} - #{padding}│ #{unclosed_delimiter(start_column)} - ... - #{end_padding}#{end_line} │ #{last_line} - #{padding}│ #{mismatched_closing_delimiter(end_column, expected_delimiter)}\ - """ - end - - defp line_range( - lines, - {start_line, start_column}, - {end_line, end_column}, - line_offset, - padding, - max_digits, - expected_delimiter - ) do - start_line = start_line - 1 - end_line = end_line - 1 - - lines - |> Enum.slice((start_line - line_offset)..(end_line - line_offset)) - |> Enum.zip_with(start_line..end_line, fn line, line_number -> - line_number = line_number + 1 - start_line = start_line + 1 - end_line = end_line + 1 - - line_padding = line_padding(line_number, max_digits) - - cond do - line_number == start_line -> - [ - line_padding, - to_string(line_number), - " │ ", - line, - "\n", - padding, - " │ ", - unclosed_delimiter(start_column) - ] - - line_number == end_line -> - [ - line_padding, - to_string(line_number), - " │ ", - line, - "\n", - padding, - " │ ", - mismatched_closing_delimiter(end_column, expected_delimiter) - ] - - true -> - [line_padding, to_string(line_number), " │ ", line] - end - end) - |> Enum.intersperse("\n") - end - - defp mismatched_closing_delimiter(end_column, expected_closing_delimiter), - do: [ - n_spaces(end_column - 1), - red(~s/└ mismatched closing delimiter (expected "#{expected_closing_delimiter}")/) - ] - - defp unclosed_delimiter(start_column), - do: [n_spaces(start_column - 1), red("└ unclosed delimiter")] - - defp pad_message(message, padding), do: String.replace(message, "\n", "\n #{padding}") - - defp red(string) do - if IO.ANSI.enabled?() do - [IO.ANSI.red(), string, IO.ANSI.reset()] - else - string - end - end - defp format_message(file, line, column, message) do location = Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) "mismatched delimiter found on " <> location <> "\n" <> message @@ -1266,8 +1275,10 @@ defmodule TokenMissingError do defexception [ :file, :line, - :snippet, :column, + :end_line, + :line_offset, + :snippet, :opening_delimiter, description: "expression is incomplete" ] @@ -1277,14 +1288,51 @@ defmodule TokenMissingError do file: file, line: line, column: column, + end_line: end_line, + line_offset: line_offset, description: description, + opening_delimiter: opening_delimiter, snippet: snippet }) - when not is_nil(snippet) and not is_nil(column) do + when not is_nil(snippet) and not is_nil(column) and not is_nil(end_line) do + {lines, total_trimmed_lines} = handle_trailing_newlines(snippet) + end_line = end_line - total_trimmed_lines + + # For cases such as inside ExUnit doctests, our snippet is tiny, containing + # only the lines in the doctest, but the `line` and `end_line` we receive + # are still tied to the whole file. + # + # In these situations we use `line_offset` to treat `line` as 1 for + # operating on the snippet, while retaining the original line information. + should_use_line_offset? = is_nil(Enum.at(lines, end_line - 1)) + + end_column = + if should_use_line_offset? do + fetch_line_length(lines, end_line - line_offset - 1) + else + fetch_line_length(lines, end_line - 1) + end + + start_pos = {line, column} + end_pos = {end_line, end_column} + expected_delimiter = :elixir_tokenizer.terminator(opening_delimiter) + + start_message = ~s/└ unclosed delimiter/ + end_message = ~s/└ missing closing delimiter (expected "#{expected_delimiter}")/ + snippet = - :elixir_errors.format_snippet({line, column}, file, description, snippet, :error, [], nil) + Exception.format_snippet( + start_pos, + end_pos, + line_offset, + description, + file, + lines, + start_message, + end_message + ) - format_message(file, line, column, snippet) + format_message(file, end_line, end_column, snippet) end @impl true @@ -1292,13 +1340,27 @@ defmodule TokenMissingError do file: file, line: line, column: column, + snippet: snippet, description: description }) do snippet = - :elixir_errors.format_snippet({line, column}, file, description, nil, :error, [], nil) + :elixir_errors.format_snippet({line, column}, file, description, snippet, :error, [], nil) - padded = " " <> String.replace(snippet, "\n", "\n ") - format_message(file, line, column, padded) + format_message(file, line, column, snippet) + end + + defp handle_trailing_newlines(snippet) do + trimmed_snippet = String.trim_trailing(snippet, "\n") + total_trimmed_newlines = String.length(snippet) - String.length(trimmed_snippet) + lines = String.split(trimmed_snippet, "\n") + {lines, total_trimmed_newlines} + end + + defp fetch_line_length(lines, index) do + lines + |> Enum.fetch!(index) + |> String.length() + |> Kernel.+(1) end defp format_message(file, line, column, message) do diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 3e3504027e1..cb0e3a17600 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -319,7 +319,10 @@ parse_error(Location, File, Error, <<>>, Input) -> <<"syntax error before: ">> -> <<"syntax error: expression is incomplete">>; _ -> <> end, - raise_snippet(Location, File, Input, 'Elixir.TokenMissingError', Message); + case lists:keytake(error_type, 1, Location) of + {value, {error_type, unclosed_delimiter}, Loc} -> raise_token_missing(Loc, File, Input, Message); + _ -> raise_snippet(Location, File, Input, 'Elixir.TokenMissingError', Message) + end; %% Show a nicer message for end of line parse_error(Location, File, <<"syntax error before: ">>, <<"eol">>, Input) -> @@ -409,6 +412,11 @@ raise_mismatched_delimiter(Location, File, Input, Message) -> KV = [{file, File}, {line_offset, StartLine - 1}, {snippet, InputBinary} | Location], raise('Elixir.MismatchedDelimiterError', Message, KV). +raise_token_missing(Location, File, Input, Message) -> + {InputString, StartLine, _} = Input, + InputBinary = elixir_utils:characters_to_binary(InputString), + raise('Elixir.TokenMissingError', Message, [{line_offset, StartLine - 1}, {file, File}, {snippet, InputBinary} | Location]). + raise_reserved(Location, File, Input, Keyword) -> raise_snippet(Location, File, Input, 'Elixir.SyntaxError', <<"syntax error before: ", Keyword/binary, ". \"", Keyword/binary, "\" is a " diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 144fdc39cfb..8e3f4746d6d 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -142,12 +142,18 @@ tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens), {ok, Line, Column, AllWarnings, AccTokens}; -tokenize([], EndLine, Column, #elixir_tokenizer{terminators=[{Start, {StartLine, _, _}, _} | _]} = Scope, Tokens) -> +tokenize([], EndLine, _, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) -> End = terminator(Start), Hint = missing_terminator_hint(Start, End, Scope), - Message = "missing terminator: ~ts (for \"~ts\" starting at line ~B)", - Formatted = io_lib:format(Message, [End, Start, StartLine]), - Meta = [{opening_delimiter, Start} | ?LOC(EndLine, Column)], + Message = "missing terminator: ~ts", + Formatted = io_lib:format(Message, [End]), + Meta = [ + {error_type, unclosed_delimiter}, + {opening_delimiter, Start}, + {line, StartLine}, + {column, StartColumn}, + {end_line, EndLine} + ], error({Meta, [Formatted, Hint], []}, [], Scope, Tokens); tokenize([], Line, Column, #elixir_tokenizer{} = Scope, Tokens) -> diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index 07b1f1a6033..184bb46a494 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -337,6 +337,140 @@ defmodule Kernel.DiagnosticsTest do end end + describe "token missing error" do + test "missing parens terminator" do + output = + capture_raise( + """ + my_numbers = [1, 2, 3, 4, 5, 6 + IO.inspect(my_numbers) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:2:23: + error: missing terminator: ] + │ + 1 │ my_numbers = [1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + 2 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected "]") + │ + └─ nofile:2:23\ + """ + end + + test "shows in between lines if EOL is not far below" do + output = + capture_raise( + """ + my_numbers = [1, 2, 3, 4, 5, 6 + my_numbers + |> Enum.map(&(&1 + 1)) + |> Enum.map(&(&1 * &1)) + |> IO.inspect() + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:5:16: + error: missing terminator: ] + │ + 1 │ my_numbers = [1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + 2 │ my_numbers + 3 │ |> Enum.map(&(&1 + 1)) + 4 │ |> Enum.map(&(&1 * &1)) + 5 │ |> IO.inspect() + │ └ missing closing delimiter (expected "]") + │ + └─ nofile:5:16\ + """ + end + + test "trims lines" do + output = + capture_raise( + """ + my_numbers = (1, 2, 3, 4, 5, 6 + + + + + + + IO.inspect(my_numbers) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:8:23: + error: missing terminator: ) + │ + 1 │ my_numbers = (1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + ... + 8 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected ")") + │ + └─ nofile:8:23\ + """ + end + + test "shows the last non-empty line of a file" do + output = + capture_raise( + """ + my_numbers = {1, 2, 3, 4, 5, 6 + IO.inspect(my_numbers) + + + + + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:2:23: + error: missing terminator: } + │ + 1 │ my_numbers = {1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + 2 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected "}") + │ + └─ nofile:2:23\ + """ + end + + test "supports unicode" do + output = + capture_raise( + """ + my_emojis = [1, 2, 3, 4 # ⚗️ + IO.inspect(my_numbers) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:2:23: + error: missing terminator: ] + │ + 1 │ my_emojis = [1, 2, 3, 4 # ⚗️ + │ └ unclosed delimiter + 2 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected "]") + │ + └─ nofile:2:23\ + """ + end + end + describe "compile-time exceptions" do test "SyntaxError (snippet)" do output = @@ -420,11 +554,16 @@ defmodule Kernel.DiagnosticsTest do """ end - test "TokenMissingError (no snippet)" do + test "TokenMissingError (unclosed delimiter)" do expected = """ - ** (TokenMissingError) token missing on nofile:2:1: - error: missing terminator: end (for "fn" starting at line 1) - └─ nofile:2:1\ + ** (TokenMissingError) token missing on nofile:1:5: + error: missing terminator: end + │ + 1 │ fn a + │ │ └ missing closing delimiter (expected "end") + │ └ unclosed delimiter + │ + └─ nofile:1:5\ """ output = diff --git a/lib/elixir/test/elixir/kernel/errors_test.exs b/lib/elixir/test/elixir/kernel/errors_test.exs index 4a54be931e5..3335b5835af 100644 --- a/lib/elixir/test/elixir/kernel/errors_test.exs +++ b/lib/elixir/test/elixir/kernel/errors_test.exs @@ -473,7 +473,7 @@ defmodule Kernel.ErrorsTest do assert_eval_raise TokenMissingError, [ "nofile:1:5:", - ~r/missing terminator: end \(for "fn" starting at line 1\)/ + ~r/missing terminator: end/ ], ~c"fn 1" @@ -490,16 +490,16 @@ defmodule Kernel.ErrorsTest do assert_eval_raise TokenMissingError, [ "nofile:1:25:", - "missing terminator: end (for \"do\" starting at line 1)", + "missing terminator: end", "defmodule ShowSnippet do\n", - "^" + "└ unclosed delimiter" ], ~c"defmodule ShowSnippet do" end test "don't show snippet when error line is empty" do assert_eval_raise TokenMissingError, - ["nofile:3:1:", "missing terminator: end (for \"do\" starting at line 1)"], + ["nofile:1:25:", "missing terminator: end"], ~c"defmodule ShowSnippet do\n\n" end diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index 149498ec2e3..e4ad788eaef 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -494,7 +494,7 @@ defmodule Kernel.ParserTest do describe "token missing errors" do test "missing paren" do assert_token_missing( - ["nofile:1:9:", "missing terminator: ) (for \"(\" starting at line 1)"], + ["nofile:1:9:", "missing terminator: )"], ~c"case 1 (" ) end @@ -549,7 +549,7 @@ defmodule Kernel.ParserTest do test "missing end" do assert_token_missing( - ["nofile:1:9:", "missing terminator: end \(for \"do\" starting at line 1\)"], + ["nofile:1:9:", "missing terminator: end"], ~c"foo do 1" ) diff --git a/lib/ex_unit/test/ex_unit/doc_test_test.exs b/lib/ex_unit/test/ex_unit/doc_test_test.exs index 7319ddbb4d6..21b3eec16ba 100644 --- a/lib/ex_unit/test/ex_unit/doc_test_test.exs +++ b/lib/ex_unit/test/ex_unit/doc_test_test.exs @@ -790,15 +790,16 @@ defmodule ExUnit.DocTestTest do line = starting_line + 35 assert output =~ """ - 6) doctest ExUnit.DocTestTest.Invalid.misplaced_opaque_type/0 (6) (ExUnit.DocTestTest.InvalidCompiled) + 6) doctest ExUnit.DocTestTest.Invalid.misplaced_opaque_type/0 (6) (ExUnit.DocTestTest.InvalidCompiled) test/ex_unit/doc_test_test.exs:#{doctest_line} - Doctest did not compile, got: (TokenMissingError) token missing on test/ex_unit/doc_test_test.exs:#{line}:7: - error: missing terminator: } (for "{" starting at line #{line}) + Doctest did not compile, got: (TokenMissingError) token missing on test/ex_unit/doc_test_test.exs:#{line}:20: + error: missing terminator: } │ - #{line} │ {:ok, #Inspect<[]>} - │ ^ + 228 │ {:ok, #Inspect<[]>} + │ │ └ missing closing delimiter (expected "}") + │ └ unclosed delimiter │ - └─ test/ex_unit/doc_test_test.exs:#{line}:7 + └─ test/ex_unit/doc_test_test.exs:#{line}:20 If you are planning to assert on the result of an iex> expression which contains a value inspected as #Name<...>, please make sure the inspected value is placed at the beginning of the expression; otherwise Elixir will treat it as a comment due to the leading sign #. doctest: iex> {:ok, :oops} From d25aacb207a2d1239966adadc754a45746ed8583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 4 Dec 2023 22:07:32 +1000 Subject: [PATCH 068/185] Simplify offset handling in TokenMissingError --- lib/elixir/lib/exception.ex | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index d6ff0ba80f3..1e50257d880 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -1298,20 +1298,11 @@ defmodule TokenMissingError do {lines, total_trimmed_lines} = handle_trailing_newlines(snippet) end_line = end_line - total_trimmed_lines - # For cases such as inside ExUnit doctests, our snippet is tiny, containing - # only the lines in the doctest, but the `line` and `end_line` we receive - # are still tied to the whole file. - # - # In these situations we use `line_offset` to treat `line` as 1 for - # operating on the snippet, while retaining the original line information. - should_use_line_offset? = is_nil(Enum.at(lines, end_line - 1)) - end_column = - if should_use_line_offset? do - fetch_line_length(lines, end_line - line_offset - 1) - else - fetch_line_length(lines, end_line - 1) - end + lines + |> Enum.fetch!(end_line - line_offset - 1) + |> String.length() + |> Kernel.+(1) start_pos = {line, column} end_pos = {end_line, end_column} @@ -1356,13 +1347,6 @@ defmodule TokenMissingError do {lines, total_trimmed_newlines} end - defp fetch_line_length(lines, index) do - lines - |> Enum.fetch!(index) - |> String.length() - |> Kernel.+(1) - end - defp format_message(file, line, column, message) do location = Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) "token missing on " <> location <> "\n" <> message From db8a1cdc7ddbfaae0a37e4e948a35745202c2e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 4 Dec 2023 22:27:47 +1000 Subject: [PATCH 069/185] Revert "Consider surround context until end whenever possible" This reverts commit a65dae971fcb44c5a845f7a8e0bbf14e3bfd2da4. --- lib/elixir/lib/code/fragment.ex | 24 +--- lib/elixir/test/elixir/code_fragment_test.exs | 104 ++++++------------ 2 files changed, 40 insertions(+), 88 deletions(-) diff --git a/lib/elixir/lib/code/fragment.ex b/lib/elixir/lib/code/fragment.ex index a95f0821902..5ad6894f9f1 100644 --- a/lib/elixir/lib/code/fragment.ex +++ b/lib/elixir/lib/code/fragment.ex @@ -636,7 +636,7 @@ defmodule Code.Fragment do {reversed_pre, post} = adjust_position(reversed_pre, post) case take_identifier(post, []) do - :none -> + {_, [], _} -> maybe_operator(reversed_pre, post, line, opts) {:identifier, reversed_post, rest} -> @@ -644,7 +644,7 @@ defmodule Code.Fragment do reversed = reversed_post ++ reversed_pre case codepoint_cursor_context(reversed, opts) do - {{:struct, acc}, offset} when acc != [] -> + {{:struct, acc}, offset} -> build_surround({:struct, acc}, reversed, line, offset) {{:alias, acc}, offset} -> @@ -749,27 +749,11 @@ defmodule Code.Fragment do do: take_identifier(t, [h | acc]) defp take_identifier(rest, acc) do - {stripped, _} = strip_spaces(rest, 0) - - with [?. | t] <- stripped, + with {[?. | t], _} <- strip_spaces(rest, 0), {[h | _], _} when h in ?A..?Z <- strip_spaces(t, 0) do take_alias(rest, acc) else - # Consider it an identifier if we are at the end of line - # or if we have spaces not followed by . (call) or / (arity) - _ when acc == [] and (rest == [] or (hd(rest) in @space and hd(stripped) not in ~c"/.")) -> - {:identifier, acc, rest} - - # If we are immediately followed by a container, we are still part of the identifier. - # We don't consider << as it _may_ be an operator. - _ when acc == [] and hd(stripped) in ~c"({[" -> - {:identifier, acc, rest} - - _ when acc == [] -> - :none - - _ -> - {:identifier, acc, rest} + _ -> {:identifier, acc, rest} end end diff --git a/lib/elixir/test/elixir/code_fragment_test.exs b/lib/elixir/test/elixir/code_fragment_test.exs index 41ad84ef405..5aa6800ae3d 100644 --- a/lib/elixir/test/elixir/code_fragment_test.exs +++ b/lib/elixir/test/elixir/code_fragment_test.exs @@ -443,12 +443,11 @@ defmodule CodeFragmentTest do end test "column out of range" do - assert CF.surround_context("hello", {1, 20}) == - %{begin: {1, 1}, context: {:local_or_var, ~c"hello"}, end: {1, 6}} + assert CF.surround_context("hello", {1, 20}) == :none end test "local_or_var" do - for i <- 1..9 do + for i <- 1..8 do assert CF.surround_context("hello_wo", {1, i}) == %{ context: {:local_or_var, ~c"hello_wo"}, begin: {1, 1}, @@ -456,9 +455,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello_wo ", {1, 10}) == :none + assert CF.surround_context("hello_wo", {1, 9}) == :none - for i <- 2..10 do + for i <- 2..9 do assert CF.surround_context(" hello_wo", {1, i}) == %{ context: {:local_or_var, ~c"hello_wo"}, begin: {1, 2}, @@ -466,9 +465,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context(" hello_wo ", {1, 11}) == :none + assert CF.surround_context(" hello_wo", {1, 10}) == :none - for i <- 1..7 do + for i <- 1..6 do assert CF.surround_context("hello!", {1, i}) == %{ context: {:local_or_var, ~c"hello!"}, begin: {1, 1}, @@ -476,9 +475,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello! ", {1, 8}) == :none + assert CF.surround_context("hello!", {1, 7}) == :none - for i <- 1..6 do + for i <- 1..5 do assert CF.surround_context("안녕_세상", {1, i}) == %{ context: {:local_or_var, ~c"안녕_세상"}, begin: {1, 1}, @@ -486,7 +485,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("안녕_세상 ", {1, 6}) == :none + assert CF.surround_context("안녕_세상", {1, 6}) == :none # Keywords are not local or var for keyword <- ~w(do end after catch else rescue fn true false nil)c do @@ -500,38 +499,8 @@ defmodule CodeFragmentTest do end end - test "local + operator" do - for i <- 1..8 do - assert CF.surround_context("hello_wo+", {1, i}) == %{ - context: {:local_or_var, ~c"hello_wo"}, - begin: {1, 1}, - end: {1, 9} - } - end - - assert CF.surround_context("hello_wo+", {1, 9}) == %{ - begin: {1, 9}, - context: {:operator, ~c"+"}, - end: {1, 10} - } - - for i <- 1..9 do - assert CF.surround_context("hello_wo +", {1, i}) == %{ - context: {:local_or_var, ~c"hello_wo"}, - begin: {1, 1}, - end: {1, 9} - } - end - - assert CF.surround_context("hello_wo +", {1, 10}) == %{ - begin: {1, 10}, - context: {:operator, ~c"+"}, - end: {1, 11} - } - end - test "local call" do - for i <- 1..9 do + for i <- 1..8 do assert CF.surround_context("hello_wo(", {1, i}) == %{ context: {:local_call, ~c"hello_wo"}, begin: {1, 1}, @@ -539,9 +508,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello_wo(", {1, 10}) == :none + assert CF.surround_context("hello_wo(", {1, 9}) == :none - for i <- 1..9 do + for i <- 1..8 do assert CF.surround_context("hello_wo (", {1, i}) == %{ context: {:local_call, ~c"hello_wo"}, begin: {1, 1}, @@ -549,10 +518,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello_wo (", {1, 10}) == :none - assert CF.surround_context("hello_wo (", {1, 11}) == :none + assert CF.surround_context("hello_wo (", {1, 9}) == :none - for i <- 1..7 do + for i <- 1..6 do assert CF.surround_context("hello!(", {1, i}) == %{ context: {:local_call, ~c"hello!"}, begin: {1, 1}, @@ -560,9 +528,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello!(", {1, 8}) == :none + assert CF.surround_context("hello!(", {1, 7}) == :none - for i <- 1..6 do + for i <- 1..5 do assert CF.surround_context("안녕_세상(", {1, i}) == %{ context: {:local_call, ~c"안녕_세상"}, begin: {1, 1}, @@ -570,7 +538,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("안녕_세상(", {1, 7}) == :none + assert CF.surround_context("안녕_세상(", {1, 6}) == :none end test "local arity" do @@ -698,7 +666,7 @@ defmodule CodeFragmentTest do end test "alias" do - for i <- 1..9 do + for i <- 1..8 do assert CF.surround_context("HelloWor", {1, i}) == %{ context: {:alias, ~c"HelloWor"}, begin: {1, 1}, @@ -706,9 +674,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("HelloWor ", {1, 10}) == :none + assert CF.surround_context("HelloWor", {1, 9}) == :none - for i <- 2..10 do + for i <- 2..9 do assert CF.surround_context(" HelloWor", {1, i}) == %{ context: {:alias, ~c"HelloWor"}, begin: {1, 2}, @@ -716,9 +684,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context(" HelloWor ", {1, 11}) == :none + assert CF.surround_context(" HelloWor", {1, 10}) == :none - for i <- 1..10 do + for i <- 1..9 do assert CF.surround_context("Hello.Wor", {1, i}) == %{ context: {:alias, ~c"Hello.Wor"}, begin: {1, 1}, @@ -726,9 +694,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("Hello.Wor ", {1, 11}) == :none + assert CF.surround_context("Hello.Wor", {1, 10}) == :none - for i <- 1..12 do + for i <- 1..11 do assert CF.surround_context("Hello . Wor", {1, i}) == %{ context: {:alias, ~c"Hello.Wor"}, begin: {1, 1}, @@ -736,9 +704,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("Hello . Wor ", {1, 13}) == :none + assert CF.surround_context("Hello . Wor", {1, 12}) == :none - for i <- 1..16 do + for i <- 1..15 do assert CF.surround_context("Foo . Bar . Baz", {1, i}) == %{ context: {:alias, ~c"Foo.Bar.Baz"}, begin: {1, 1}, @@ -770,7 +738,7 @@ defmodule CodeFragmentTest do end: {1, 11} } - for i <- 1..15 do + for i <- 1..14 do assert CF.surround_context("__MODULE__.Foo", {1, i}) == %{ context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"}, begin: {1, 1}, @@ -778,7 +746,7 @@ defmodule CodeFragmentTest do } end - for i <- 1..19 do + for i <- 1..18 do assert CF.surround_context("__MODULE__.Foo.Sub", {1, i}) == %{ context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"}, begin: {1, 1}, @@ -830,7 +798,7 @@ defmodule CodeFragmentTest do end test "attribute submodules" do - for i <- 1..10 do + for i <- 1..9 do assert CF.surround_context("@some.Foo", {1, i}) == %{ context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo"}, begin: {1, 1}, @@ -838,7 +806,7 @@ defmodule CodeFragmentTest do } end - for i <- 1..14 do + for i <- 1..13 do assert CF.surround_context("@some.Foo.Sub", {1, i}) == %{ context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"}, begin: {1, 1}, @@ -921,7 +889,7 @@ defmodule CodeFragmentTest do end: {1, 15} } - for i <- 2..10 do + for i <- 2..9 do assert CF.surround_context("%HelloWor", {1, i}) == %{ context: {:struct, ~c"HelloWor"}, begin: {1, 1}, @@ -929,7 +897,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("%HelloWor ", {1, 11}) == :none + assert CF.surround_context("%HelloWor", {1, 10}) == :none # With dot assert CF.surround_context("%Hello.Wor", {1, 1}) == %{ @@ -938,7 +906,7 @@ defmodule CodeFragmentTest do end: {1, 11} } - for i <- 2..11 do + for i <- 2..10 do assert CF.surround_context("%Hello.Wor", {1, i}) == %{ context: {:struct, ~c"Hello.Wor"}, begin: {1, 1}, @@ -946,7 +914,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("%Hello.Wor ", {1, 12}) == :none + assert CF.surround_context("%Hello.Wor", {1, 11}) == :none # With spaces assert CF.surround_context("% Hello . Wor", {1, 1}) == %{ @@ -955,7 +923,7 @@ defmodule CodeFragmentTest do end: {1, 14} } - for i <- 2..14 do + for i <- 2..13 do assert CF.surround_context("% Hello . Wor", {1, i}) == %{ context: {:struct, ~c"Hello.Wor"}, begin: {1, 1}, @@ -963,7 +931,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("% Hello . Wor ", {1, 15}) == :none + assert CF.surround_context("% Hello . Wor", {1, 14}) == :none end test "module attributes" do From e1db6d8831c1f61bbb774b07d656d48c2b4d4453 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Wed, 6 Dec 2023 22:43:53 +0100 Subject: [PATCH 070/185] Update Windows installer to register in Add/Remove Programs (#13156) --- .../assets/{drop.ico => Elixir.ico} | Bin lib/elixir/scripts/windows_installer/build.sh | 38 +++++------------- .../scripts/windows_installer/installer.nsi | 12 +++++- 3 files changed, 22 insertions(+), 28 deletions(-) rename lib/elixir/scripts/windows_installer/assets/{drop.ico => Elixir.ico} (100%) diff --git a/lib/elixir/scripts/windows_installer/assets/drop.ico b/lib/elixir/scripts/windows_installer/assets/Elixir.ico similarity index 100% rename from lib/elixir/scripts/windows_installer/assets/drop.ico rename to lib/elixir/scripts/windows_installer/assets/Elixir.ico diff --git a/lib/elixir/scripts/windows_installer/build.sh b/lib/elixir/scripts/windows_installer/build.sh index e9b51e91f77..117fc517721 100755 --- a/lib/elixir/scripts/windows_installer/build.sh +++ b/lib/elixir/scripts/windows_installer/build.sh @@ -4,43 +4,27 @@ # With Elixir archive: # # ELIXIR_ZIP=Precompiled.zip OTP_VERSION=25.3.2.2 ./build.sh -# -# With Elixir version: -# -# ELIXIR_VERSION=1.14.5 OTP_VERSION=25.3.2.2 ./build.sh set -euo pipefail -OTP_VERSION="${OTP_VERSION:-26.0}" -otp_release=`echo "${OTP_VERSION}" | cut -d. -f1` - mkdir -p tmp +rm -rf tmp/elixir +unzip -d "tmp/elixir" "${ELIXIR_ZIP}" -ELIXIR_VERSION="${ELIXIR_VERSION:-}" -if [ -n "${ELIXIR_VERSION}" ]; then - ELIXIR_ZIP="tmp/elixir-${ELIXIR_VERSION}-otp-${otp_release}.zip" - if [ ! -f "${ELIXIR_ZIP}" ]; then - url="https://github.com/elixir-lang/elixir/releases/download/v${ELIXIR_VERSION}/elixir-otp-${otp_release}.zip" - echo "downloading ${url}" - curl --fail -L -o "${ELIXIR_ZIP}" "${url}" - fi - basename=elixir-${ELIXIR_VERSION}-otp-${otp_release} -else - basename=elixir-otp-${otp_release} -fi - -if [ ! -d "tmp/${basename}" ]; then - unzip -d "tmp/${basename}" "${ELIXIR_ZIP}" -fi +elixir_version=`cat tmp/elixir/VERSION` +otp_release=`erl -noshell -eval 'io:put_chars(erlang:system_info(otp_release)), halt().'` +otp_version=`erl -noshell -eval '{ok, Vsn} = file:read_file(code:root_dir() ++ "/releases/" ++ erlang:system_info(otp_release) ++ "/OTP_VERSION"), io:put_chars(Vsn), halt().'` +elixir_exe=elixir-otp-${otp_release}.exe # brew install makensis # apt install -y nsis # choco install -y nsis export PATH="/c/Program Files (x86)/NSIS:${PATH}" makensis \ - -X"OutFile tmp\\${basename}.exe" \ - -DOTP_VERSION=${OTP_VERSION} \ + -X"OutFile tmp\\${elixir_exe}" \ -DOTP_RELEASE="${otp_release}" \ - -DELIXIR_DIR=tmp\\${basename} \ + -DOTP_VERSION=${otp_version} \ + -DELIXIR_DIR=tmp\\elixir \ + -DELIXIR_VERSION=${elixir_version} \ installer.nsi -echo "Installer path: tmp/${basename}.exe" +echo "Installer path: tmp/${elixir_exe}" diff --git a/lib/elixir/scripts/windows_installer/installer.nsi b/lib/elixir/scripts/windows_installer/installer.nsi index a9e8495ea67..8df40aad159 100644 --- a/lib/elixir/scripts/windows_installer/installer.nsi +++ b/lib/elixir/scripts/windows_installer/installer.nsi @@ -6,7 +6,8 @@ Name "Elixir" ManifestDPIAware true Unicode True InstallDir "$PROGRAMFILES64\Elixir" -!define MUI_ICON "assets\drop.ico" +!define MUI_ICON "assets\Elixir.ico" +!define MUI_UNICON "assets\Elixir.ico" ; Install Page: Install Erlang/OTP @@ -191,7 +192,15 @@ FunctionEnd Section "Install Elixir" SectionElixir SetOutPath "$INSTDIR" File /r "${ELIXIR_DIR}\" + File "assets\Elixir.ico" File "update_system_path.erl" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "DisplayName" "Elixir" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "DisplayVersion" "${ELIXIR_VERSION}" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "DisplayIcon" "$INSTDIR\Elixir.ico" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "Publisher" "The Elixir Team" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "UninstallString" '"$INSTDIR\Uninstall.exe"' + WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "NoModify" 1 + WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "NoRepair" 1 WriteUninstaller "Uninstall.exe" SectionEnd @@ -270,6 +279,7 @@ UninstPage custom un.FinishPageShow un.FinishPageLeave Section "Uninstall" RMDir /r "$INSTDIR" + DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" SectionEnd !insertmacro MUI_LANGUAGE "English" From 4e4cde1118b9d02fb75ef0579fdc5be37a8f7818 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 7 Dec 2023 00:28:04 +0100 Subject: [PATCH 071/185] Update Windows installer to write Elixir install root to registry (#13157) We don't need this right now but it could be useful in the future, if anything to detect if Elixir was installed using this installer. Demo: iex> {:ok, r} = :win32reg.open([:read]) iex> :win32reg.change_key(r, ~c"\\hklm\\software\\wow6432node\\elixir\\elixir") iex> :win32reg.value(r, ~c"installroot") {:ok, ~c"C:\\Program Files\\Elixir"} --- lib/elixir/scripts/windows_installer/installer.nsi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/elixir/scripts/windows_installer/installer.nsi b/lib/elixir/scripts/windows_installer/installer.nsi index 8df40aad159..d7d4c9bae94 100644 --- a/lib/elixir/scripts/windows_installer/installer.nsi +++ b/lib/elixir/scripts/windows_installer/installer.nsi @@ -202,6 +202,8 @@ Section "Install Elixir" SectionElixir WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "NoModify" 1 WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "NoRepair" 1 + WriteRegStr HKLM "Software\Elixir\Elixir" "InstallRoot" "$INSTDIR" + WriteUninstaller "Uninstall.exe" SectionEnd @@ -280,6 +282,8 @@ UninstPage custom un.FinishPageShow un.FinishPageLeave Section "Uninstall" RMDir /r "$INSTDIR" DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" + DeleteRegKey HKLM "Software\Elixir\Elixir" + DeleteRegKey /ifempty HKLM "Software\Elixir" SectionEnd !insertmacro MUI_LANGUAGE "English" From 3b5bd39743328f7f83e060a111591dbfcaeefae8 Mon Sep 17 00:00:00 2001 From: Himanshu <79717090+Himanshu21git@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:28:51 +0530 Subject: [PATCH 072/185] Update case-cond-and-if.md (#13158) --- lib/elixir/pages/getting-started/case-cond-and-if.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/getting-started/case-cond-and-if.md b/lib/elixir/pages/getting-started/case-cond-and-if.md index 431ff2786e7..a1406653267 100644 --- a/lib/elixir/pages/getting-started/case-cond-and-if.md +++ b/lib/elixir/pages/getting-started/case-cond-and-if.md @@ -65,7 +65,7 @@ iex> case :ok do ** (CaseClauseError) no case clause matching: :ok ``` -The documentation for the `Kernel` module lists all available guards in its sidebar. You can also consult the complete [Patterns and Guards](../references/patterns-and-guards.html#guards) reference for in-depth documentation. +The documentation for the `Kernel` module lists all available guards in its sidebar. You can also consult the complete [Patterns and Guards](../references/patterns-and-guards.md#guards) reference for in-depth documentation. ## cond From c4dcbf379fcb07997171ea8817f22e6798d0e3e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 9 Dec 2023 20:39:38 +1100 Subject: [PATCH 073/185] Pass original exception down to details in diagnostic, closes #13142 --- lib/elixir/lib/code.ex | 1 + lib/elixir/lib/kernel/parallel_compiler.ex | 6 +++++- lib/mix/lib/mix/compilers/elixir.ex | 21 ++++++++++++--------- lib/mix/lib/mix/task.compiler.ex | 2 +- lib/mix/test/mix/tasks/compile_test.exs | 3 ++- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 2d6a0d02f11..320bf457437 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -204,6 +204,7 @@ defmodule Code do required(:position) => position, required(:stacktrace) => Exception.stacktrace(), required(:span) => {non_neg_integer, non_neg_integer} | nil, + optional(:exception) => Exception.t() | nil, optional(any()) => any() } diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index b86d8dc38fe..ee5e3c7d3ec 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -900,10 +900,14 @@ defmodule Kernel.ParallelCompiler do message: message, severity: :error, stacktrace: stack, - span: nil + span: nil, + exception: get_exception(reason) } end + defp get_exception(exception) when is_exception(exception), do: exception + defp get_exception(_reason), do: nil + defp get_line(_file, %{line: line, column: column}, _stack) when is_integer(line) and line > 0 and is_integer(column) and column >= 0 do {line, column} diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex index 7f7bbb32988..22e2686affc 100644 --- a/lib/mix/lib/mix/compilers/elixir.ex +++ b/lib/mix/lib/mix/compilers/elixir.ex @@ -767,14 +767,16 @@ defmodule Mix.Compilers.Elixir do end end - defp diagnostic(%{ - file: file, - position: position, - message: message, - severity: severity, - stacktrace: stacktrace, - span: span - }) do + defp diagnostic( + %{ + file: file, + position: position, + message: message, + severity: severity, + stacktrace: stacktrace, + span: span + } = diagnostic + ) do %Mix.Task.Compiler.Diagnostic{ file: file, position: position, @@ -782,7 +784,8 @@ defmodule Mix.Compilers.Elixir do severity: severity, compiler_name: "Elixir", stacktrace: stacktrace, - span: span + span: span, + details: Map.get(diagnostic, :exception, nil) } end diff --git a/lib/mix/lib/mix/task.compiler.ex b/lib/mix/lib/mix/task.compiler.ex index be968b2ee25..9c80400f814 100644 --- a/lib/mix/lib/mix/task.compiler.ex +++ b/lib/mix/lib/mix/task.compiler.ex @@ -39,7 +39,7 @@ defmodule Mix.Task.Compiler do message: IO.chardata(), position: position, compiler_name: String.t(), - details: any, + details: Exception.t() | any, stacktrace: Exception.stacktrace(), span: {non_neg_integer, non_neg_integer} | nil } diff --git a/lib/mix/test/mix/tasks/compile_test.exs b/lib/mix/test/mix/tasks/compile_test.exs index ccc969257f4..cb73f993a26 100644 --- a/lib/mix/test/mix/tasks/compile_test.exs +++ b/lib/mix/test/mix/tasks/compile_test.exs @@ -182,7 +182,8 @@ defmodule Mix.Tasks.CompileTest do severity: :error, position: {2, 20}, message: "** (SyntaxError) invalid syntax found on lib/a.ex:2:" <> _, - compiler_name: "Elixir" + compiler_name: "Elixir", + details: %SyntaxError{} } = diagnostic end) end) From b1998960bb7a576fa210589ef2e6330ffb42aab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 9 Dec 2023 20:49:12 +1100 Subject: [PATCH 074/185] Disable compiler optimizations only in module body --- lib/elixir/src/elixir_compiler.erl | 12 ++++++------ lib/elixir/src/elixir_module.erl | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/elixir/src/elixir_compiler.erl b/lib/elixir/src/elixir_compiler.erl index 487d3b3fa55..980aa9e0e86 100644 --- a/lib/elixir/src/elixir_compiler.erl +++ b/lib/elixir/src/elixir_compiler.erl @@ -1,6 +1,6 @@ %% Elixir compiler front-end to the Erlang backend. -module(elixir_compiler). --export([string/3, quoted/3, bootstrap/0, file/2, compile/3]). +-export([string/3, quoted/3, bootstrap/0, file/2, compile/4]). -include("elixir.hrl"). string(Contents, File, Callback) -> @@ -36,22 +36,22 @@ maybe_fast_compile(Forms, Args, E) -> case (?key(E, module) == nil) andalso allows_fast_compilation(Forms) andalso (not elixir_config:is_bootstrap()) of true -> fast_compile(Forms, E); - false -> compile(Forms, Args, E) + false -> compile(Forms, Args, [], E) end, ok. -compile(Quoted, ArgsList, #{line := Line} = E) -> +compile(Quoted, ArgsList, CompilerOpts, #{line := Line} = E) -> Block = no_tail_optimize([{line, Line}], Quoted), {Expanded, SE, EE} = elixir_expand:expand(Block, elixir_env:env_to_ex(E), E), elixir_env:check_unused_vars(SE, EE), {Module, Fun, Purgeable} = - elixir_erl_compiler:spawn(fun() -> spawned_compile(Expanded, E) end), + elixir_erl_compiler:spawn(fun() -> spawned_compile(Expanded, CompilerOpts, E) end), Args = list_to_tuple(ArgsList), {dispatch(Module, Fun, Args, Purgeable), SE, EE}. -spawned_compile(ExExprs, #{line := Line, file := File} = E) -> +spawned_compile(ExExprs, CompilerOpts, #{line := Line, file := File} = E) -> {Vars, S} = elixir_erl_var:from_env(E), {ErlExprs, _} = elixir_erl_pass:translate(ExExprs, erl_anno:new(Line), S), @@ -59,7 +59,7 @@ spawned_compile(ExExprs, #{line := Line, file := File} = E) -> Fun = code_fun(?key(E, module)), Forms = code_mod(Fun, ErlExprs, Line, File, Module, Vars), - {Module, Binary} = elixir_erl_compiler:noenv_forms(Forms, File, [nowarn_nomatch, no_bool_opt, no_ssa_opt]), + {Module, Binary} = elixir_erl_compiler:noenv_forms(Forms, File, [nowarn_nomatch | CompilerOpts]), code:load_binary(Module, "", Binary), {Module, Fun, is_purgeable(Module, Binary)}. diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index 1cee92e4a1e..637598d6f27 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -421,7 +421,9 @@ build(Module, Line, File, E) -> %% Handles module and callback evaluations. eval_form(Line, Module, DataBag, Block, Vars, Prune, E) -> - {Value, ExS, EE} = elixir_compiler:compile(Block, Vars, E), + %% Given Elixir modules can get very long to compile due to metaprogramming, + %% we disable expansions that take linear time to code size. + {Value, ExS, EE} = elixir_compiler:compile(Block, Vars, [no_bool_opt, no_ssa_opt], E), elixir_overridable:store_not_overridden(Module), EV = (elixir_env:reset_vars(EE))#{line := Line}, EC = eval_callbacks(Line, DataBag, before_compile, [EV], EV), From 2bfb14751f5c382a7eedc620763e7b4d0974e456 Mon Sep 17 00:00:00 2001 From: Daven <77761194+fmterrorf@users.noreply.github.com> Date: Sun, 10 Dec 2023 01:58:10 -0600 Subject: [PATCH 075/185] Add warning when deps clean fails (#13161) --- lib/mix/lib/mix/tasks/deps.clean.ex | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/mix/lib/mix/tasks/deps.clean.ex b/lib/mix/lib/mix/tasks/deps.clean.ex index 1fbdb41e743..24810465013 100644 --- a/lib/mix/lib/mix/tasks/deps.clean.ex +++ b/lib/mix/lib/mix/tasks/deps.clean.ex @@ -99,6 +99,27 @@ defmodule Mix.Tasks.Deps.Clean do paths end + defp maybe_warn_failed_file_deletion(results, dependency) when is_list(results) do + messages = + Enum.flat_map(results, fn + {:error, reason, file} -> + ["\tfile: #{file}, reason: #{:file.format_error(reason)}"] + + _ -> + [] + end) + + with [_ | _] <- messages do + Mix.shell().error( + "warning: errors occurred while deleting files for dependency: #{dependency} \n" <> + Enum.join(messages, "\n") + ) + end + end + + defp maybe_warn_failed_file_deletion(result, dependency), + do: maybe_warn_failed_file_deletion([result], dependency) + defp do_clean(apps, deps, build_path, deps_path, build_only?) do shell = Mix.shell() @@ -112,7 +133,8 @@ defmodule Mix.Tasks.Deps.Clean do |> Path.join(to_string(app)) |> Path.wildcard() |> maybe_warn_for_invalid_path(app) - |> Enum.each(&File.rm_rf!/1) + |> Enum.map(&File.rm_rf/1) + |> maybe_warn_failed_file_deletion(app) # Remove everything from the source directory of dependencies. # Skip this step if --build option is specified or if @@ -122,7 +144,8 @@ defmodule Mix.Tasks.Deps.Clean do else deps_path |> Path.join(to_string(app)) - |> File.rm_rf!() + |> File.rm_rf() + |> maybe_warn_failed_file_deletion(app) end end) end From 5b6e04cb4d68e0e0a5402eeff5206c4d2bff2dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 10 Dec 2023 09:03:41 +0100 Subject: [PATCH 076/185] Clean up failed deletion warning --- lib/mix/lib/mix/tasks/deps.clean.ex | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/lib/mix/lib/mix/tasks/deps.clean.ex b/lib/mix/lib/mix/tasks/deps.clean.ex index 24810465013..efb8bd5c43d 100644 --- a/lib/mix/lib/mix/tasks/deps.clean.ex +++ b/lib/mix/lib/mix/tasks/deps.clean.ex @@ -99,27 +99,15 @@ defmodule Mix.Tasks.Deps.Clean do paths end - defp maybe_warn_failed_file_deletion(results, dependency) when is_list(results) do - messages = - Enum.flat_map(results, fn - {:error, reason, file} -> - ["\tfile: #{file}, reason: #{:file.format_error(reason)}"] - - _ -> - [] - end) - - with [_ | _] <- messages do + defp maybe_warn_failed_file_deletion(result) do + with {:error, reason, file} <- result do Mix.shell().error( - "warning: errors occurred while deleting files for dependency: #{dependency} \n" <> - Enum.join(messages, "\n") + "warning: could not delete file #{Path.relative_to_cwd(file)}, " <> + "reason: #{:file.format_error(reason)}" ) end end - defp maybe_warn_failed_file_deletion(result, dependency), - do: maybe_warn_failed_file_deletion([result], dependency) - defp do_clean(apps, deps, build_path, deps_path, build_only?) do shell = Mix.shell() @@ -133,8 +121,7 @@ defmodule Mix.Tasks.Deps.Clean do |> Path.join(to_string(app)) |> Path.wildcard() |> maybe_warn_for_invalid_path(app) - |> Enum.map(&File.rm_rf/1) - |> maybe_warn_failed_file_deletion(app) + |> Enum.map(&(&1 |> File.rm_rf() |> maybe_warn_failed_file_deletion())) # Remove everything from the source directory of dependencies. # Skip this step if --build option is specified or if @@ -145,7 +132,7 @@ defmodule Mix.Tasks.Deps.Clean do deps_path |> Path.join(to_string(app)) |> File.rm_rf() - |> maybe_warn_failed_file_deletion(app) + |> maybe_warn_failed_file_deletion() end end) end From 6475917b3f25649a4177503ddb4e012001d2537d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 11 Dec 2023 07:28:35 +0100 Subject: [PATCH 077/185] Use Macro.Env to record attribute warnings Closes #13162. Closes #13164. --- lib/elixir/lib/module.ex | 3 +- .../test/elixir/kernel/warning_test.exs | 35 ++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/lib/elixir/lib/module.ex b/lib/elixir/lib/module.ex index 087c04f1a00..30a9f7d09b9 100644 --- a/lib/elixir/lib/module.ex +++ b/lib/elixir/lib/module.ex @@ -2133,8 +2133,7 @@ defmodule Module do end defp attribute_stack(module, line) do - file = String.to_charlist(Path.relative_to_cwd(:elixir_module.file(module))) - [{module, :__MODULE__, 0, file: file, line: line}] + struct!(Macro.Env, module: module, file: :elixir_module.file(module), line: line) end ## Helpers diff --git a/lib/elixir/test/elixir/kernel/warning_test.exs b/lib/elixir/test/elixir/kernel/warning_test.exs index 518da3c4c85..5196bd0b4e7 100644 --- a/lib/elixir/test/elixir/kernel/warning_test.exs +++ b/lib/elixir/test/elixir/kernel/warning_test.exs @@ -1640,22 +1640,39 @@ defmodule Kernel.WarningTest do end test "reserved doc metadata keys" do - output = - capture_eval(""" - defmodule Sample do - @typedoc opaque: false - @type t :: binary + {output, diagnostics} = + Code.with_diagnostics([log: true], fn -> + capture_eval(""" + defmodule Sample do + @typedoc opaque: false + @type t :: binary - @doc defaults: 3, since: "1.2.3" - def foo(a), do: a - end - """) + @doc defaults: 3, since: "1.2.3" + def foo(a), do: a + end + """) + end) assert output =~ "ignoring reserved documentation metadata key: :opaque" assert output =~ "nofile:2: " assert output =~ "ignoring reserved documentation metadata key: :defaults" assert output =~ "nofile:5: " refute output =~ ":since" + + assert [ + %{ + message: "ignoring reserved documentation metadata key: :opaque", + position: 2, + file: "nofile", + severity: :warning + }, + %{ + message: "ignoring reserved documentation metadata key: :defaults", + position: 5, + file: "nofile", + severity: :warning + } + ] = diagnostics after purge(Sample) end From 88ba7766e0ff7a3ac4975dee92bc04f6e0045eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 11 Dec 2023 11:13:24 +0100 Subject: [PATCH 078/185] Use Macro.Env in more warnings --- lib/elixir/lib/kernel.ex | 2 +- lib/elixir/lib/kernel/utils.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/elixir/lib/kernel.ex b/lib/elixir/lib/kernel.ex index 41859e34ff3..146d3ab5b14 100644 --- a/lib/elixir/lib/kernel.ex +++ b/lib/elixir/lib/kernel.ex @@ -3622,7 +3622,7 @@ defmodule Kernel do defp do_at([], meta, name, function?, env) do IO.warn( "the @#{name}() notation (with parentheses) is deprecated, please use @#{name} (without parentheses) instead", - Macro.Env.stacktrace(env) + env ) do_at(nil, meta, name, function?, env) diff --git a/lib/elixir/lib/kernel/utils.ex b/lib/elixir/lib/kernel/utils.ex index 23235f11704..43d1a74f667 100644 --- a/lib/elixir/lib/kernel/utils.ex +++ b/lib/elixir/lib/kernel/utils.ex @@ -36,14 +36,14 @@ defmodule Kernel.Utils do if is_list(funs) do IO.warn( "passing a list to Kernel.defdelegate/2 is deprecated, please define each delegate separately", - Macro.Env.stacktrace(env) + env ) end if Keyword.has_key?(opts, :append_first) do IO.warn( "Kernel.defdelegate/2 :append_first option is deprecated", - Macro.Env.stacktrace(env) + env ) end From 81a12b777411e5fb3dc48cb314771c03202e0679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 12 Dec 2023 12:36:20 +0100 Subject: [PATCH 079/185] Release v1.16.0-rc.1 --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- bin/elixir | 2 +- bin/elixir.bat | 2 +- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c728bf541e6..cc81ba96338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,49 @@ Another [ExDoc](https://github.com/elixir-lang/ex_doc) feature we have incorpora Finally, we have started enriching our documentation with [Mermaid.js](https://mermaid.js.org/) diagrams. You can find examples in the [GenServer](https://hexdocs.pm/elixir/main/GenServer.html) and [Supervisor](https://hexdocs.pm/elixir/main/Supervisor.html) docs. +## v1.16.0-rc.1 (2023-12-12) + +### 1. Enhancements + +#### Elixir + + * [Code] Add `:emit_warnings` for `Code.string_to_quoted/2` + * [File] Add `:offset` option to `File.stream!/2` + * [Kernel] Auto infer size of matched variable in bitstrings + * [Kernel] Preserve column information when translating typespecs + * [String] Add `String.replace_invalid/2` + +#### Logger + + * [Logger] Add `Logger.levels/0` + +#### Mix + + * [mix archive.install] Support `--sparse` option + * [mix compile.app] Warn if both `:applications` and `:extra_applications` are used + * [mix compile.elixir] Pass original exception down to diagnostic `:details` when possible + * [mix deps.clean] Emit a warning instead of crashing when a dependency cannot be removed + * [mix escript.install] Support `--sparse` option + * [mix release] Include `include/` directory in releases + +### 2. Bug fixes + +#### Elixir + + * [Code] Keep quotes for atom keys in formatter + * [Macro] Address exception on `Macro.to_string/1` for certain ASTs + * [Module] Make sure file and position information is included in several module warnings (regression) + * [Path] Lazily evaluate `File.cwd!/0` in `Path.expand/1` and `Path.absname/1` + +#### IEx + + * [IEx.Pry] Fix prying functions with only literals in their body + +#### Mix + + * [mix archive.install] Restore code paths after archive.install + * [mix escript.install] Restore code paths after archive.install + ## v1.16.0-rc.0 (2023-10-31) ### 1. Enhancements diff --git a/VERSION b/VERSION index b6ea925546e..f45d77ad4ce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.0-rc.0 +1.16.0-rc.1 \ No newline at end of file diff --git a/bin/elixir b/bin/elixir index ffa46a15047..01259f98449 100755 --- a/bin/elixir +++ b/bin/elixir @@ -1,7 +1,7 @@ #!/bin/sh set -e -ELIXIR_VERSION=1.16.0-rc.0 +ELIXIR_VERSION=1.16.0-rc.1 if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then cat <&2 diff --git a/bin/elixir.bat b/bin/elixir.bat index 26594925d79..e52fa8db9e7 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -1,6 +1,6 @@ @if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) -set ELIXIR_VERSION=1.16.0-rc.0 +set ELIXIR_VERSION=1.16.0-rc.1 setlocal enabledelayedexpansion if ""%1""=="""" if ""%2""=="""" goto documentation From a09ddbb0a637382473a6e5c05dedcdd94238d87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 12 Dec 2023 12:43:34 +0100 Subject: [PATCH 080/185] Fix typo in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc81ba96338..ed540654950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,7 +104,7 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m #### Mix * [mix archive.install] Restore code paths after archive.install - * [mix escript.install] Restore code paths after archive.install + * [mix escript.install] Restore code paths after escript.install ## v1.16.0-rc.0 (2023-10-31) From 9675e2285aad6c865fb6802eee69fbd5917a51d5 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Tue, 12 Dec 2023 14:31:12 +0100 Subject: [PATCH 081/185] elixir.bat: Quote file paths (#13172) --- bin/elixir.bat | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/elixir.bat b/bin/elixir.bat index e52fa8db9e7..f29bfe0a190 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -140,16 +140,16 @@ if ""==!par:--remsh=! (set "parsElixir=!parsElixir! --remsh %~1" && shift && if ""==!par:--dot-iex=! (set "parsElixir=!parsElixir! --dot-iex %~1" && shift && goto startloop) if ""==!par:--dbg=! (set "parsElixir=!parsElixir! --dbg %~1" && shift && goto startloop) rem ******* ERLANG PARAMETERS ********************** -if ""==!par:--boot=! (set "parsErlang=!parsErlang! -boot %~1" && shift && goto startloop) -if ""==!par:--boot-var=! (set "parsErlang=!parsErlang! -boot_var %~1 %~2" && shift && shift && goto startloop) -if ""==!par:--cookie=! (set "parsErlang=!parsErlang! -setcookie %~1" && shift && goto startloop) +if ""==!par:--boot=! (set "parsErlang=!parsErlang! -boot "%~1"" && shift && goto startloop) +if ""==!par:--boot-var=! (set "parsErlang=!parsErlang! -boot_var "%~1" "%~2"" && shift && shift && goto startloop) +if ""==!par:--cookie=! (set "parsErlang=!parsErlang! -setcookie "%~1"" && shift && goto startloop) if ""==!par:--hidden=! (set "parsErlang=!parsErlang! -hidden" && goto startloop) -if ""==!par:--erl-config=! (set "parsErlang=!parsErlang! -config %~1" && shift && goto startloop) +if ""==!par:--erl-config=! (set "parsErlang=!parsErlang! -config "%~1"" && shift && goto startloop) if ""==!par:--logger-otp-reports=! (set "parsErlang=!parsErlang! -logger handle_otp_reports %1" && shift && goto startloop) if ""==!par:--logger-sasl-reports=! (set "parsErlang=!parsErlang! -logger handle_sasl_reports %1" && shift && goto startloop) -if ""==!par:--name=! (set "parsErlang=!parsErlang! -name %~1" && shift && goto startloop) -if ""==!par:--sname=! (set "parsErlang=!parsErlang! -sname %~1" && shift && goto startloop) -if ""==!par:--vm-args=! (set "parsErlang=!parsErlang! -args_file %~1" && shift && goto startloop) +if ""==!par:--name=! (set "parsErlang=!parsErlang! -name "%~1"" && shift && goto startloop) +if ""==!par:--sname=! (set "parsErlang=!parsErlang! -sname "%~1"" && shift && goto startloop) +if ""==!par:--vm-args=! (set "parsErlang=!parsErlang! -args_file "%~1"" && shift && goto startloop) if ""==!par:--erl=! (set "beforeExtra=!beforeExtra! %~1" && shift && goto startloop) if ""==!par:--pipe-to=! (echo --pipe-to : Option is not supported on Windows && goto end) set endLoop=1 From 1070b2f43445fda640c7797df2e70950e79e0eaf Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Tue, 12 Dec 2023 16:43:38 +0100 Subject: [PATCH 082/185] Mention dangers around `Task` and sending a lot of data along (#13173) The `Task` module is one of the coolest modules in elixir and is probably the first contact and experience of a lot of beginners with parallelism in elixir. I hence find it worthwhile to warn about the memory copying and its impacts here as it might easily lead to unwelcome results, so it's worth pointing out. --- lib/elixir/lib/task.ex | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/elixir/lib/task.ex b/lib/elixir/lib/task.ex index 2dd53da2a66..1b221050faa 100644 --- a/lib/elixir/lib/task.ex +++ b/lib/elixir/lib/task.ex @@ -44,10 +44,38 @@ defmodule Task do means that, if the caller crashes, the task will crash too and vice-versa. This is on purpose: if the process meant to receive the result no longer exists, there is - no purpose in completing the computation. + no purpose in completing the computation. If this is not + desired, you will want to use supervised tasks, described + in a subsequent section. - If this is not desired, you will want to use supervised - tasks, described next. + ## Tasks are processes + + Tasks are processes and so data will need to be completely copied + to them. Take the following code as an example: + + large_data = fetch_large_data() + task = Task.async(fn -> do_some_work(large_data) end) + res = do_some_other_work() + res + Task.await(task) + + The code above copies over all of `large_data`, which can be + resource intensive depending on the size of the data. + There are two ways to address this. + + First, if you need to access only part of `large_data`, + consider extracting it before the task: + + large_data = fetch_large_data() + subset_data = large_data.some_field + task = Task.async(fn -> do_some_work(subset_data) end) + + Alternatively, if you can move the data loading altogether + to the task, it may be even better: + + task = Task.async(fn -> + large_data = fetch_large_data() + do_some_work(large_data) + end) ## Dynamically supervised tasks From fb42733873a7fe85c2f4a271ba00dce041d39a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 13 Dec 2023 06:43:20 +0100 Subject: [PATCH 083/185] Improve anonymous functions guide --- lib/elixir/pages/getting-started/anonymous-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/getting-started/anonymous-functions.md b/lib/elixir/pages/getting-started/anonymous-functions.md index a584b3d63af..8c12cf242a6 100644 --- a/lib/elixir/pages/getting-started/anonymous-functions.md +++ b/lib/elixir/pages/getting-started/anonymous-functions.md @@ -17,7 +17,7 @@ true In the example above, we defined an anonymous function that receives two arguments, `a` and `b`, and returns the result of `a + b`. The arguments are always on the left-hand side of `->` and the code to be executed on the right-hand side. The anonymous function is stored in the variable `add`. -We can invoke anonymous functions by passing arguments to it. Note that a dot (`.`) between the variable and parentheses is required to invoke an anonymous function. The dot ensures there is no ambiguity between calling the anonymous function matched to a variable `add` and a named function `add/2`. We will write our own named functions when dealing with [Modules and Functions](modules-and-functions.md). For now, just remember that Elixir makes a clear distinction between anonymous functions and named functions. +We can invoke anonymous functions by passing arguments to it. Note that a dot (`.`) between the variable and parentheses is required to invoke an anonymous function. The dot makes it clear when you are calling an anonymous function, stored in the variable `add`, opposed to a function named `add/2`. For example, if you have an anonymous function stored in the variable `is_atom`, there is no ambiguity between `is_atom.(:foo)` and `is_atom(:foo)`. If both used the same `is_atom(:foo)` syntax, the only way to know the actual behaviour of `is_atom(:foo)` would be by scanning all code thus far for a possible definition of the `is_atom` variable. This scanning hurts maintainability as it requires developers to track additional context in their head when reading and writing code. Anonymous functions in Elixir are also identified by the number of arguments they receive. We can check if a function is of any given arity by using `is_function/2`: From 010d516b11cfbd737c1c60727ec728eab23d22b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 13 Dec 2023 11:56:41 +0100 Subject: [PATCH 084/185] Fix indentation in mix release --- lib/mix/lib/mix/tasks/release.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/mix/lib/mix/tasks/release.ex b/lib/mix/lib/mix/tasks/release.ex index 1c08674824d..e9d13d58f08 100644 --- a/lib/mix/lib/mix/tasks/release.ex +++ b/lib/mix/lib/mix/tasks/release.ex @@ -389,10 +389,10 @@ defmodule Mix.Tasks.Release do * `:strip_beams` - controls if BEAM files should have their debug information, documentation chunks, and other non-essential metadata removed. Defaults to `true`. May be set to `false` to disable stripping. Also accepts - `[keep: ["Docs", "Dbgi"]]` to keep certain chunks that are usually stripped. - You can also set the `:compress` option to true to enable individual - compression of BEAM files, although it is typically preferred to compress - the whole release instead. + `[keep: ["Docs", "Dbgi"]]` to keep certain chunks that are usually stripped. + You can also set the `:compress` option to true to enable individual + compression of BEAM files, although it is typically preferred to compress + the whole release instead. * `:cookie` - a string representing the Erlang Distribution cookie. If this option is not set, a random cookie is written to the `releases/COOKIE` file From 80404da8f6c214729fcbceac09194a069c6a9b0c Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Wed, 13 Dec 2023 14:36:33 +0100 Subject: [PATCH 085/185] Mention the fixed regression (#13176) As best as I learned this was introduced in #11420 and fixed in aabe465 It shouldn't affect almost anyone except for scripting usage of elixir and (probably most notably) benchee benchmarks that don't call functions in a module. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed540654950..40cf15f2dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m #### Elixir * [Code] Keep quotes for atom keys in formatter + * [Kernel] Re-enabled compiler optimzations for top level functions in scripts (enabled in v1.14.0 but shouldn't impact most programs) * [Macro] Address exception on `Macro.to_string/1` for certain ASTs * [Module] Make sure file and position information is included in several module warnings (regression) * [Path] Lazily evaluate `File.cwd!/0` in `Path.expand/1` and `Path.absname/1` From c950a57a3e1bbd0714d6b0a38fea030acfdfadf8 Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Wed, 13 Dec 2023 14:56:19 +0100 Subject: [PATCH 086/185] Fix top level functions changelog (enabled --> disabled) (#13177) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40cf15f2dab..cedaf69ca83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,7 +93,7 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m #### Elixir * [Code] Keep quotes for atom keys in formatter - * [Kernel] Re-enabled compiler optimzations for top level functions in scripts (enabled in v1.14.0 but shouldn't impact most programs) + * [Kernel] Re-enabled compiler optimzations for top level functions in scripts (disabled in v1.14.0 but shouldn't impact most programs) * [Macro] Address exception on `Macro.to_string/1` for certain ASTs * [Module] Make sure file and position information is included in several module warnings (regression) * [Path] Lazily evaluate `File.cwd!/0` in `Path.expand/1` and `Path.absname/1` From a50a2fd9836843d27f581a4ebb536f15c250e8b2 Mon Sep 17 00:00:00 2001 From: Maarten van Vliet Date: Wed, 13 Dec 2023 23:17:11 +0100 Subject: [PATCH 087/185] Fix typo (#13181) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cedaf69ca83..31ee1cdac59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,7 +93,7 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m #### Elixir * [Code] Keep quotes for atom keys in formatter - * [Kernel] Re-enabled compiler optimzations for top level functions in scripts (disabled in v1.14.0 but shouldn't impact most programs) + * [Kernel] Re-enabled compiler optimizations for top level functions in scripts (disabled in v1.14.0 but shouldn't impact most programs) * [Macro] Address exception on `Macro.to_string/1` for certain ASTs * [Module] Make sure file and position information is included in several module warnings (regression) * [Path] Lazily evaluate `File.cwd!/0` in `Path.expand/1` and `Path.absname/1` From 88bbffd61453eb05429fb73ddf759cc17ce0fe44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20M=C3=BCller?= Date: Wed, 13 Dec 2023 19:31:38 -0300 Subject: [PATCH 088/185] Add info about unclosed delimiters diagnostic to 1.16 changelog (#13180) --- CHANGELOG.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31ee1cdac59..2f604724701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,21 @@ For mismatched delimiters, it now shows both delimiters: └─ lib/my_app.ex:1:18 ``` +For unclosed delimiters, it now shows where the unclosed delimiter starts: + +``` +** (TokenMissingError) token missing on lib/my_app:8:23: + error: missing terminator: ) + │ + 1 │ my_numbers = (1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + ... + 8 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected ")") + │ + └─ lib/my_app:8:23 +``` + Errors and warnings diagnostics also include code snippets. When possible, we will show precise spans, such as on undefined variables: ``` @@ -51,7 +66,7 @@ error: function names should start with lowercase characters or underscore, inva └─ lib/sample.ex:3 ``` -A huge thank you to Vinícius Muller for working on the new diagnostics. +A huge thank you to Vinícius Müller for working on the new diagnostics. ## Revamped documentation From 1878e30bdb15c05d6d1dae3c4e5e5841051037eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 13 Dec 2023 23:30:54 +0100 Subject: [PATCH 089/185] Improve docs and support column in IO.warn, closes #13179 --- lib/elixir/lib/io.ex | 34 ++++++++++++------- .../test/elixir/kernel/diagnostics_test.exs | 26 +++++++++++++- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/lib/elixir/lib/io.ex b/lib/elixir/lib/io.ex index d5d0788b6d3..42b3a03025b 100644 --- a/lib/elixir/lib/io.ex +++ b/lib/elixir/lib/io.ex @@ -308,17 +308,20 @@ defmodule IO do entry from the compilation environment will be used * a keyword list with at least the `:file` option representing - a single stacktrace entry (since v1.14.0). The `:line`, `:module`, - `:function` options are also supported + a single stacktrace entry (since v1.14.0). The `:line`, `:column`, + `:module`, and `:function` options are also supported - This function also notifies the compiler a warning was printed - (in case --warnings-as-errors was enabled). It returns `:ok` - if it succeeds. + This function notifies the compiler a warning was printed + and emits a compiler diagnostic (`t:Code.diagnostic/1`). + The diagnostic will include precise file and location information + if a `Macro.Env` is given or those values have been passed as + keyword list, but not for stacktraces, as they are often imprecise. + + It returns `:ok` if it succeeds. ## Examples - stacktrace = [{MyApp, :main, 1, [file: 'my_app.ex', line: 4]}] - IO.warn("variable bar is unused", stacktrace) + IO.warn("variable bar is unused", module: MyApp, function: {:main, 1}, line: 4, file: "my_app.ex") #=> warning: variable bar is unused #=> my_app.ex:4: MyApp.main/1 @@ -337,15 +340,22 @@ defmodule IO do def warn(message, [{_, _} | _] = keyword) do if file = keyword[:file] do - warn( - message, - %{ + line = keyword[:line] + column = keyword[:column] + position = if line && column, do: {line, column}, else: line + message = to_chardata(message) + + stacktrace = + Macro.Env.stacktrace(%{ __ENV__ | module: keyword[:module], function: keyword[:function], - line: keyword[:line], + line: line, file: file - } + }) + + :elixir_errors.emit_diagnostic(:warning, position, file, message, stacktrace, + read_snippet: true ) else warn(message, []) diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index 184bb46a494..be5f0e042e2 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -769,6 +769,30 @@ defmodule Kernel.DiagnosticsTest do purge(Sample) end + @tag :tmp_dir + test "IO.warn file+line", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "io-warn-file-line.ex") + + source = """ + IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line) + """ + + File.write!(path, source) + + expected = """ + warning: oops + multi + line + │ + 1 │ IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line) + │ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + │ + └─ tmp\ + """ + + assert capture_io(:stderr, fn -> Code.eval_file(path) end) =~ expected + end + @tag :tmp_dir test "IO.warn file+line+column", %{tmp_dir: tmp_dir} do path = make_relative_tmp(tmp_dir, "io-warn-file-line-column.ex") @@ -785,7 +809,7 @@ defmodule Kernel.DiagnosticsTest do line │ 1 │ IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line, column: 4) - │ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + │ ~ │ └─ tmp\ """ From 67fd9824cc8991a70d2d889962014bde2ae6b3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20M=C3=BCller?= Date: Thu, 14 Dec 2023 05:24:43 -0300 Subject: [PATCH 090/185] Improve diagnostics for unclosed heredocs (#13182) --- lib/elixir/lib/exception.ex | 17 +++++++++--- lib/elixir/src/elixir_tokenizer.erl | 12 ++++++++- .../test/elixir/kernel/diagnostics_test.exs | 27 +++++++++++++++++++ lib/elixir/test/elixir/kernel/parser_test.exs | 2 +- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index 1e50257d880..41459d3de1c 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -794,6 +794,15 @@ defmodule Exception do end end + @doc false + def format_expected_delimiter(opening_delimiter) do + terminator = :elixir_tokenizer.terminator(opening_delimiter) + + if terminator |> Atom.to_string() |> String.contains?(["\"", "'"]), + do: terminator, + else: ~s("#{terminator}") + end + @doc false def format_snippet( {start_line, _start_column} = start_pos, @@ -1183,10 +1192,10 @@ defmodule MismatchedDelimiterError do start_pos = {start_line, start_column} end_pos = {end_line, end_column} lines = String.split(snippet, "\n") - expected_delimiter = :elixir_tokenizer.terminator(opening_delimiter) + expected_delimiter = Exception.format_expected_delimiter(opening_delimiter) start_message = "└ unclosed delimiter" - end_message = ~s/└ mismatched closing delimiter (expected "#{expected_delimiter}")/ + end_message = ~s/└ mismatched closing delimiter (expected #{expected_delimiter})/ snippet = Exception.format_snippet( @@ -1306,10 +1315,10 @@ defmodule TokenMissingError do start_pos = {line, column} end_pos = {end_line, end_column} - expected_delimiter = :elixir_tokenizer.terminator(opening_delimiter) + expected_delimiter = Exception.format_expected_delimiter(opening_delimiter) start_message = ~s/└ unclosed delimiter/ - end_message = ~s/└ missing closing delimiter (expected "#{expected_delimiter}")/ + end_message = ~s/└ missing closing delimiter (expected #{expected_delimiter})/ snippet = Exception.format_snippet( diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 8e3f4746d6d..e4f3f512229 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -1030,7 +1030,16 @@ extract_heredoc_with_interpolation(Line, Column, Scope, Interpol, T, H) -> {ok, NewLine, NewColumn, tokens_to_binary(Parts2), Rest, NewScope}; {error, Reason} -> - {error, interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line])} + {Position, Message, List} = interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line]), + {line, EndLine} = lists:keyfind(line, 1, Position), + Meta = [ + {error_type, unclosed_delimiter}, + {opening_delimiter, '"""'}, + {line, Line}, + {column, Column}, + {end_line, EndLine} + ], + {error, {Meta, Message, List}} end; error -> @@ -1481,6 +1490,7 @@ terminator('do') -> 'end'; terminator('(') -> ')'; terminator('[') -> ']'; terminator('{') -> '}'; +terminator('"""') -> '"""'; terminator('<<') -> '>>'. %% Keywords checking diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index be5f0e042e2..bf177ebfc9d 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -361,6 +361,33 @@ defmodule Kernel.DiagnosticsTest do """ end + test "missing heredoc terminator" do + output = + capture_raise( + """ + a = \""" + test string + + IO.inspect(10 + 20) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:4:20: + error: missing terminator: \""" (for heredoc starting at line 1) + │ + 1 │ a = \""" + │ └ unclosed delimiter + 2 │ test string + 3 │ + 4 │ IO.inspect(10 + 20) + │ └ missing closing delimiter (expected \""") + │ + └─ nofile:4:20\ + """ + end + test "shows in between lines if EOL is not far below" do output = capture_raise( diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index e4ad788eaef..4e30410b495 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -528,7 +528,7 @@ defmodule Kernel.ParserTest do test "heredoc with incomplete interpolation" do assert_token_missing( [ - "nofile:2:1:", + "nofile:1:4:", ~s/missing interpolation terminator: "}" (for heredoc starting at line 1)/ ], ~c"\"\"\"\n\#{\n" From 7abb3bdddc6d4459385464ba011402cc7bda6591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 14 Dec 2023 11:14:40 +0100 Subject: [PATCH 091/185] Unify position handling and improve docs See #13179. See #13184. --- lib/elixir/lib/code.ex | 23 +++++++++++++++++++---- lib/mix/lib/mix/task.compiler.ex | 26 +++++++++----------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 320bf457437..1945ce2e2c1 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -196,21 +196,36 @@ defmodule Code do @typedoc """ Diagnostics returned by the compiler and code evaluation. + + If there is a file and position, then the diagnostic is precise + and you can use the given file and position for generating snippets, + IDEs annotations, and so on. + + Otherwise, a stacktrace may be given, which you can place your own + heuristics to provide better reporting. """ @type diagnostic(severity) :: %{ - required(:file) => Path.t(), + required(:file) => Path.t() | nil, required(:severity) => severity, required(:message) => String.t(), - required(:position) => position, + required(:position) => position(), required(:stacktrace) => Exception.stacktrace(), - required(:span) => {non_neg_integer, non_neg_integer} | nil, + required(:span) => {non_neg_integer(), non_neg_integer()} | nil, optional(:exception) => Exception.t() | nil, optional(any()) => any() } @typedoc "The line. 0 indicates no line." @type line() :: non_neg_integer() - @type position() :: line() | {pos_integer(), column :: non_neg_integer} + + @typedoc """ + The position of the diagnostic. + + Can be either a line number or a `{line, column}`. + Line and columns numbers are one-based. + A position of `0` represents unknown. + """ + @type position() :: line() | {line :: pos_integer(), column :: pos_integer()} @boolean_compiler_options [ :docs, diff --git a/lib/mix/lib/mix/task.compiler.ex b/lib/mix/lib/mix/task.compiler.ex index 9c80400f814..619eb2eb54c 100644 --- a/lib/mix/lib/mix/task.compiler.ex +++ b/lib/mix/lib/mix/task.compiler.ex @@ -31,13 +31,20 @@ defmodule Mix.Task.Compiler do defmodule Diagnostic do @moduledoc """ Diagnostic information such as a warning or compilation error. + + If there is a file and position, then the diagnostic is precise + and you can use the given file and position for generating snippets, + IDEs annotations, and so on. + + Otherwise, a stacktrace may be given, which you can place your own + heuristics to provide better reporting. """ @type t :: %__MODULE__{ - file: Path.t(), + file: Path.t() | nil, severity: severity, message: IO.chardata(), - position: position, + position: Code.position(), compiler_name: String.t(), details: Exception.t() | any, stacktrace: Exception.stacktrace(), @@ -61,21 +68,6 @@ defmodule Mix.Task.Compiler do """ @type severity :: :error | :warning | :information | :hint - @typedoc """ - Where in a file the diagnostic applies. Can be either a line number, - a `{line, column}` tuple, a range specified as `{start_line, start_col, - end_line, end_col}`. `0` line represents unknown. - - Line numbers are one-based, and column numbers in a range are zero-based and refer - to the cursor position at the start of the character at that index. For example, - to indicate that a diagnostic applies to the first `n` characters of the - first line, the range would be `{1, 0, 1, n}`. - """ - @type position :: - non_neg_integer - | {pos_integer, non_neg_integer} - | {pos_integer, non_neg_integer, pos_integer, non_neg_integer} - @enforce_keys [:file, :severity, :message, :position, :compiler_name] defstruct [ :file, From 2e61f32b6dd0f4cafa4d561de85cead7ab6ec9c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 14 Dec 2023 11:23:56 +0100 Subject: [PATCH 092/185] Update CHANGELOG --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f604724701..939c33df786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,7 +170,8 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m #### Mix - * [Mix] Ensure files with duplicate modules are recompiled whenever any of the files change + * [mix compile] Ensure files with duplicate modules are recompiled whenever any of the files change + * [mix compile] Update Mix compiler diagnostics documentation and typespecs to match the Elixir compiler behaviour where both lines and columns start from one (before it inaccurately said that columns started from zero) ### 3. Soft deprecations (no warnings emitted) @@ -180,6 +181,10 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m * [Kernel.ParallelCompiler] Deprecate `Kernel.ParallelCompiler.async/1` in favor of `Kernel.ParallelCompiler.pmap/2` * [Path] Deprecate `Path.safe_relative_to/2` in favor of `Path.safe_relative/2` +#### Mix + + * [mix compile] Returning a four-element tuple as a position in `Mix.Task.Compiler.Diagnostic` + ### 4. Hard deprecations #### Elixir From 872efb180da056b08d8311d369ae1da51f884ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 14 Dec 2023 12:07:20 +0100 Subject: [PATCH 093/185] Document diagnostic span --- lib/elixir/lib/code.ex | 5 +++-- lib/mix/lib/mix/task.compiler.ex | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 1945ce2e2c1..25ac1ca37b6 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -199,7 +199,8 @@ defmodule Code do If there is a file and position, then the diagnostic is precise and you can use the given file and position for generating snippets, - IDEs annotations, and so on. + IDEs annotations, and so on. An optional span is available with + the line and column the diagnostic ends. Otherwise, a stacktrace may be given, which you can place your own heuristics to provide better reporting. @@ -210,7 +211,7 @@ defmodule Code do required(:message) => String.t(), required(:position) => position(), required(:stacktrace) => Exception.stacktrace(), - required(:span) => {non_neg_integer(), non_neg_integer()} | nil, + required(:span) => {line :: pos_integer(), column :: pos_integer()} | nil, optional(:exception) => Exception.t() | nil, optional(any()) => any() } diff --git a/lib/mix/lib/mix/task.compiler.ex b/lib/mix/lib/mix/task.compiler.ex index 619eb2eb54c..44055a710f9 100644 --- a/lib/mix/lib/mix/task.compiler.ex +++ b/lib/mix/lib/mix/task.compiler.ex @@ -34,7 +34,8 @@ defmodule Mix.Task.Compiler do If there is a file and position, then the diagnostic is precise and you can use the given file and position for generating snippets, - IDEs annotations, and so on. + IDEs annotations, and so on. An optional span is available with + the line and column the diagnostic ends. Otherwise, a stacktrace may be given, which you can place your own heuristics to provide better reporting. @@ -48,7 +49,7 @@ defmodule Mix.Task.Compiler do compiler_name: String.t(), details: Exception.t() | any, stacktrace: Exception.stacktrace(), - span: {non_neg_integer, non_neg_integer} | nil + span: {line :: pos_integer(), column :: pos_integer()} | nil } @typedoc """ From d23e42e27feb1ea88152d0a36b0f89804b18436b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 14 Dec 2023 16:58:55 +0100 Subject: [PATCH 094/185] Normalize token missing and mismatched delimiter exceptions Closes #13183. Closes #13185. Closes #13186. Closes #13187. --- lib/elixir/lib/exception.ex | 56 ++++++++----- lib/elixir/lib/kernel/parallel_compiler.ex | 52 +++++++----- lib/elixir/src/elixir_tokenizer.erl | 60 +++++++------- lib/elixir/test/elixir/code_test.exs | 10 ++- .../test/elixir/kernel/diagnostics_test.exs | 80 ++++++++++++++++++- lib/elixir/test/elixir/kernel/parser_test.exs | 77 ------------------ 6 files changed, 187 insertions(+), 148 deletions(-) diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index 41459d3de1c..bbdeefee3d6 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -795,12 +795,10 @@ defmodule Exception do end @doc false - def format_expected_delimiter(opening_delimiter) do - terminator = :elixir_tokenizer.terminator(opening_delimiter) - - if terminator |> Atom.to_string() |> String.contains?(["\"", "'"]), - do: terminator, - else: ~s("#{terminator}") + def format_delimiter(delimiter) do + if delimiter |> Atom.to_string() |> String.contains?(["\"", "'"]), + do: delimiter, + else: ~s("#{delimiter}") end @doc false @@ -1159,8 +1157,23 @@ defmodule MismatchedDelimiterError do An exception raised when a mismatched delimiter is found when parsing code. For example: - - `[1, 2, 3}` - - `fn a -> )` + + * `[1, 2, 3}` + * `fn a -> )` + + The following fields of this exceptions are public and can be accessed freely: + + * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if + the error occurred in code that did not come from a file + * `:line` - the line for the opening delimiter + * `:column` - the column for the opening delimiter + * `:end_line` - the line for the mismatched closing delimiter + * `:end_column` - the column for the mismatched closing delimiter + * `:opening_delimiter` - an atom representing the opening delimiter + * `:closing_delimiter` - an atom representing the mismatched closing delimiter + * `:expected_delimiter` - an atom representing the closing delimiter + * `:description` - a description of the mismatched delimiter error + """ defexception [ @@ -1172,6 +1185,7 @@ defmodule MismatchedDelimiterError do :end_column, :opening_delimiter, :closing_delimiter, + :expected_delimiter, :snippet, description: "mismatched delimiter error" ] @@ -1184,15 +1198,14 @@ defmodule MismatchedDelimiterError do end_column: end_column, line_offset: line_offset, description: description, - opening_delimiter: opening_delimiter, - closing_delimiter: _closing_delimiter, + expected_delimiter: expected_delimiter, file: file, snippet: snippet }) do start_pos = {start_line, start_column} end_pos = {end_line, end_column} lines = String.split(snippet, "\n") - expected_delimiter = Exception.format_expected_delimiter(opening_delimiter) + expected_delimiter = Exception.format_delimiter(expected_delimiter) start_message = "└ unclosed delimiter" end_message = ~s/└ mismatched closing delimiter (expected #{expected_delimiter})/ @@ -1226,8 +1239,9 @@ defmodule SyntaxError do * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if the error occurred in code that did not come from a file - * `:line` (`t:non_neg_integer/0`) - the line where the error occurred - * `:column` (`t:non_neg_integer/0`) - the column where the error occurred + * `:line` - the line where the error occurred + * `:column` - the column where the error occurred + * `:description` - a description of the syntax error """ @@ -1276,9 +1290,13 @@ defmodule TokenMissingError do * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if the error occurred in code that did not come from a file - * `:line` (`t:non_neg_integer/0`) - the line where the error occurred - * `:column` (`t:non_neg_integer/0`) - the column where the error occurred - + * `:line` - the line for the opening delimiter + * `:column` - the column for the opening delimiter + * `:end_line` - the line for the end of the string + * `:end_column` - the column for the end of the string + * `:opening_delimiter` - an atom representing the opening delimiter + * `:expected_delimiter` - an atom representing the expected delimiter + * `:description` - a description of the missing token error """ defexception [ @@ -1286,9 +1304,11 @@ defmodule TokenMissingError do :line, :column, :end_line, + :end_column, :line_offset, :snippet, :opening_delimiter, + :expected_delimiter, description: "expression is incomplete" ] @@ -1300,7 +1320,7 @@ defmodule TokenMissingError do end_line: end_line, line_offset: line_offset, description: description, - opening_delimiter: opening_delimiter, + expected_delimiter: expected_delimiter, snippet: snippet }) when not is_nil(snippet) and not is_nil(column) and not is_nil(end_line) do @@ -1315,7 +1335,7 @@ defmodule TokenMissingError do start_pos = {line, column} end_pos = {end_line, end_column} - expected_delimiter = Exception.format_expected_delimiter(opening_delimiter) + expected_delimiter = Exception.format_delimiter(expected_delimiter) start_message = ~s/└ unclosed delimiter/ end_message = ~s/└ missing closing delimiter (expected #{expected_delimiter})/ diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index ee5e3c7d3ec..580e60d3482 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -890,7 +890,7 @@ defmodule Kernel.ParallelCompiler do end defp to_error(file, kind, reason, stack) do - line = get_line(file, reason, stack) + {line, span} = get_line_span(file, reason, stack) file = Path.absname(file) message = :unicode.characters_to_binary(Kernel.CLI.format_error(kind, reason, stack)) @@ -900,7 +900,7 @@ defmodule Kernel.ParallelCompiler do message: message, severity: :error, stacktrace: stack, - span: nil, + span: span, exception: get_exception(reason) } end @@ -908,35 +908,47 @@ defmodule Kernel.ParallelCompiler do defp get_exception(exception) when is_exception(exception), do: exception defp get_exception(_reason), do: nil - defp get_line(_file, %{line: line, column: column}, _stack) + defp get_line_span( + _file, + %{line: line, column: column, end_line: end_line, end_column: end_column}, + _stack + ) + when is_integer(line) and line > 0 and is_integer(column) and column >= 0 and + is_integer(end_line) and end_line > 0 and is_integer(end_column) and end_column >= 0 do + {{line, column}, {end_line, end_column}} + end + + defp get_line_span(_file, %{line: line, column: column}, _stack) when is_integer(line) and line > 0 and is_integer(column) and column >= 0 do - {line, column} + {{line, column}, nil} end - defp get_line(_file, %{line: line}, _stack) when is_integer(line) and line > 0 do - line + defp get_line_span(_file, %{line: line}, _stack) when is_integer(line) and line > 0 do + {line, nil} end - defp get_line(file, :undef, [{_, _, _, []}, {_, _, _, info} | _]) do - if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do - Keyword.get(info, :line) - end + defp get_line_span(file, :undef, [{_, _, _, []}, {_, _, _, info} | _]) do + get_line_span_from_stacktrace_info(info, file) end - defp get_line(file, _reason, [{_, _, _, [file: expanding]}, {_, _, _, info} | _]) + defp get_line_span(file, _reason, [{_, _, _, [file: expanding]}, {_, _, _, info} | _]) when expanding in [~c"expanding macro", ~c"expanding struct"] do - if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do - Keyword.get(info, :line) - end + get_line_span_from_stacktrace_info(info, file) end - defp get_line(file, _reason, [{_, _, _, info} | _]) do - if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do - Keyword.get(info, :line) - end + defp get_line_span(file, _reason, [{_, _, _, info} | _]) do + get_line_span_from_stacktrace_info(info, file) + end + + defp get_line_span(_, _, _) do + {nil, nil} end - defp get_line(_, _, _) do - nil + defp get_line_span_from_stacktrace_info(info, file) do + if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do + {Keyword.get(info, :line), nil} + else + {nil, nil} + end end end diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index e4f3f512229..54c3dcc19d9 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -142,18 +142,20 @@ tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens), {ok, Line, Column, AllWarnings, AccTokens}; -tokenize([], EndLine, _, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) -> +tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) -> End = terminator(Start), Hint = missing_terminator_hint(Start, End, Scope), Message = "missing terminator: ~ts", Formatted = io_lib:format(Message, [End]), Meta = [ - {error_type, unclosed_delimiter}, - {opening_delimiter, Start}, - {line, StartLine}, - {column, StartColumn}, - {end_line, EndLine} - ], + {error_type, unclosed_delimiter}, + {opening_delimiter, Start}, + {expected_delimiter, End}, + {line, StartLine}, + {column, StartColumn}, + {end_line, EndLine}, + {end_column, EndColumn} + ], error({Meta, [Formatted, Hint], []}, [], Scope, Tokens); tokenize([], Line, Column, #elixir_tokenizer{} = Scope, Tokens) -> @@ -531,7 +533,7 @@ tokenize([$:, H | T] = Original, Line, Column, Scope, Tokens) when ?is_quote(H) {error, Reason} -> Message = " (for atom starting at line ~B)", - interpolation_error(Reason, Original, Scope, Tokens, Message, [Line]) + interpolation_error(Reason, Original, Scope, Tokens, Message, [Line], Line, Column + 1, [H], [H]) end; tokenize([$: | String] = Original, Line, Column, Scope, Tokens) -> @@ -772,7 +774,7 @@ handle_heredocs(T, Line, Column, H, Scope, Tokens) -> handle_strings(T, Line, Column, H, Scope, Tokens) -> case elixir_interpolation:extract(Line, Column, Scope, true, T, H) of {error, Reason} -> - interpolation_error(Reason, [H | T], Scope, Tokens, " (for string starting at line ~B)", [Line]); + interpolation_error(Reason, [H | T], Scope, Tokens, " (for string starting at line ~B)", [Line], Line, Column-1, [H], [H]); {NewLine, NewColumn, Parts, [$: | Rest], InterScope} when ?is_space(hd(Rest)) -> NewScope = case is_unnecessary_quote(Parts, InterScope) of @@ -925,7 +927,7 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, Scope, Tokens) when ?i Message = "interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: ", error({?LOC(Line, Column), Message, [H]}, Rest, NewScope, Tokens); {error, Reason} -> - interpolation_error(Reason, Original, Scope, Tokens, " (for function name starting at line ~B)", [Line]) + interpolation_error(Reason, Original, Scope, Tokens, " (for function name starting at line ~B)", [Line], Line, Column, [H], [H]) end; handle_dot([$. | Rest], Line, Column, DotInfo, Scope, Tokens) -> @@ -1030,16 +1032,7 @@ extract_heredoc_with_interpolation(Line, Column, Scope, Interpol, T, H) -> {ok, NewLine, NewColumn, tokens_to_binary(Parts2), Rest, NewScope}; {error, Reason} -> - {Position, Message, List} = interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line]), - {line, EndLine} = lists:keyfind(line, 1, Position), - Meta = [ - {error_type, unclosed_delimiter}, - {opening_delimiter, '"""'}, - {line, Line}, - {column, Column}, - {end_line, EndLine} - ], - {error, {Meta, Message, List}} + {error, interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line], Line, Column, [H, H, H], [H, H, H])} end; error -> @@ -1352,12 +1345,21 @@ previous_was_eol(_) -> nil. %% Error handling -interpolation_error(Reason, Rest, Scope, Tokens, Extension, Args) -> - error(interpolation_format(Reason, Extension, Args), Rest, Scope, Tokens). +interpolation_error(Reason, Rest, Scope, Tokens, Extension, Args, Line, Column, Opening, Closing) -> + error(interpolation_format(Reason, Extension, Args, Line, Column, Opening, Closing), Rest, Scope, Tokens). -interpolation_format({string, Line, Column, Message, Token}, Extension, Args) -> - {?LOC(Line, Column), [Message, io_lib:format(Extension, Args)], Token}; -interpolation_format({_, _, _} = Reason, _Extension, _Args) -> +interpolation_format({string, EndLine, EndColumn, Message, Token}, Extension, Args, Line, Column, Opening, Closing) -> + Meta = [ + {error_type, unclosed_delimiter}, + {opening_delimiter, list_to_atom(Opening)}, + {expected_delimiter, list_to_atom(Closing)}, + {line, Line}, + {column, Column}, + {end_line, EndLine}, + {end_column, EndColumn} + ], + {Meta, [Message, io_lib:format(Extension, Args)], Token}; +interpolation_format({_, _, _} = Reason, _Extension, _Args, _Line, _Column, _Opening, _Closing) -> Reason. %% Terminators @@ -1429,7 +1431,7 @@ check_terminator({End, {EndLine, EndColumn, _}}, [{Start, {StartLine, StartColum End -> {ok, Scope#elixir_tokenizer{terminators=Terminators}}; - _ExpectedEnd -> + ExpectedEnd -> Meta = [ {line, StartLine}, {column, StartColumn}, @@ -1437,7 +1439,8 @@ check_terminator({End, {EndLine, EndColumn, _}}, [{Start, {StartLine, StartColum {end_column, EndColumn}, {error_type, mismatched_delimiter}, {opening_delimiter, Start}, - {closing_delimiter, End} + {closing_delimiter, End}, + {expected_delimiter, ExpectedEnd} ], {error, {Meta, unexpected_token_or_reserved(End), [atom_to_list(End)]}} end; @@ -1490,7 +1493,6 @@ terminator('do') -> 'end'; terminator('(') -> ')'; terminator('[') -> ']'; terminator('{') -> '}'; -terminator('"""') -> '"""'; terminator('<<') -> '>>'. %% Keywords checking @@ -1596,7 +1598,7 @@ tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, S {error, Reason} -> Sigil = [$~, S, H], Message = " (for sigil ~ts starting at line ~B)", - interpolation_error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens, Message, [Sigil, Line]) + interpolation_error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens, Message, [Sigil, Line], Line, Column, [H], [sigil_terminator(H)]) end; tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Tokens) -> diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index b9be631cb74..de6733d99d3 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -498,9 +498,13 @@ defmodule CodeTest do end test "string_to_quoted returns error on incomplete escaped string" do - assert Code.string_to_quoted("\"\\") == - {:error, - {[line: 1, column: 3], "missing terminator: \" (for string starting at line 1)", ""}} + assert {:error, {meta, "missing terminator: \" (for string starting at line 1)", ""}} = + Code.string_to_quoted("\"\\") + + assert meta[:line] == 1 + assert meta[:column] == 1 + assert meta[:end_line] == 1 + assert meta[:end_column] == 3 end test "compile source" do diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index bf177ebfc9d..4a057180b9b 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -380,7 +380,7 @@ defmodule Kernel.DiagnosticsTest do 1 │ a = \""" │ └ unclosed delimiter 2 │ test string - 3 │ + 3 │\s 4 │ IO.inspect(10 + 20) │ └ missing closing delimiter (expected \""") │ @@ -388,6 +388,84 @@ defmodule Kernel.DiagnosticsTest do """ end + test "missing sigil terminator" do + output = + capture_raise("~s (for sigil ~s< starting at line 1) + │ + 1 │ ~s") + │ └ unclosed delimiter + │ + └─ nofile:1:10\ + """ + + output = + capture_raise("~s|foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:10: + error: missing terminator: | (for sigil ~s| starting at line 1) + │ + 1 │ ~s|foobar + │ │ └ missing closing delimiter (expected "|") + │ └ unclosed delimiter + │ + └─ nofile:1:10\ + """ + end + + test "missing string terminator" do + output = + capture_raise("\"foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:8: + error: missing terminator: " (for string starting at line 1) + │ + 1 │ "foobar + │ │ └ missing closing delimiter (expected ") + │ └ unclosed delimiter + │ + └─ nofile:1:8\ + """ + end + + test "missing atom terminator" do + output = + capture_raise(":\"foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:9: + error: missing terminator: " (for atom starting at line 1) + │ + 1 │ :"foobar + │ │ └ missing closing delimiter (expected ") + │ └ unclosed delimiter + │ + └─ nofile:1:9\ + """ + end + + test "missing function terminator" do + output = + capture_raise("K.\"foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:10: + error: missing terminator: " (for function name starting at line 1) + │ + 1 │ K."foobar + │ │ └ missing closing delimiter (expected ") + │ └ unclosed delimiter + │ + └─ nofile:1:10\ + """ + end + test "shows in between lines if EOL is not far below" do output = capture_raise( diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index 4e30410b495..b1803d0e511 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -491,83 +491,6 @@ defmodule Kernel.ParserTest do end end - describe "token missing errors" do - test "missing paren" do - assert_token_missing( - ["nofile:1:9:", "missing terminator: )"], - ~c"case 1 (" - ) - end - - test "dot terminator" do - assert_token_missing( - ["nofile:1:9:", "missing terminator: \" (for function name starting at line 1)"], - ~c"foo.\"bar" - ) - end - - test "sigil terminator" do - assert_token_missing( - ["nofile:3:1:", "missing terminator: \" (for sigil ~r\" starting at line 1)"], - ~c"~r\"foo\n\n" - ) - - assert_token_missing( - ["nofile:3:1:", "missing terminator: } (for sigil ~r{ starting at line 1)"], - ~c"~r{foo\n\n" - ) - end - - test "string terminator" do - assert_token_missing( - ["nofile:1:5:", "missing terminator: \" (for string starting at line 1)"], - ~c"\"bar" - ) - end - - test "heredoc with incomplete interpolation" do - assert_token_missing( - [ - "nofile:1:4:", - ~s/missing interpolation terminator: "}" (for heredoc starting at line 1)/ - ], - ~c"\"\"\"\n\#{\n" - ) - end - - test "heredoc terminator" do - assert_token_missing( - ["nofile:2:4:", ~s/missing terminator: """ (for heredoc starting at line 1)/], - ~c"\"\"\"\nbar" - ) - - assert_token_missing( - ["nofile:2:7:", ~s/missing terminator: """ (for heredoc starting at line 1)/], - ~c"\"\"\"\nbar\"\"\"" - ) - end - - test "missing end" do - assert_token_missing( - ["nofile:1:9:", "missing terminator: end"], - ~c"foo do 1" - ) - - assert_token_missing( - ["hint:", ~s/it looks like the "do" on line 2 does not have a matching "end"/], - ~c""" - defmodule MyApp do - def one do - # end - - def two do - end - end - """ - ) - end - end - describe "syntax errors" do test "invalid heredoc start" do assert_syntax_error( From d13f4155afb603125e1684e4b5556489fe78e9bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 14 Dec 2023 17:18:13 +0100 Subject: [PATCH 095/185] Do not warn unused imports twice, closes #13178 --- lib/elixir/src/elixir_import.erl | 36 +++++++++++--------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/lib/elixir/src/elixir_import.erl b/lib/elixir/src/elixir_import.erl index 520e8bcbc87..0f824a96a78 100644 --- a/lib/elixir/src/elixir_import.erl +++ b/lib/elixir/src/elixir_import.erl @@ -9,24 +9,26 @@ import(Meta, Ref, Opts, E) -> {Functions, Macros, Added} = case keyfind(only, Opts) of {only, functions} -> - {Added1, Funs} = import_functions(Meta, Ref, Opts, E), + {Added1, _Used1, Funs} = import_functions(Meta, Ref, Opts, E), {Funs, keydelete(Ref, ?key(E, macros)), Added1}; {only, macros} -> - {Added2, Macs} = import_macros(true, Meta, Ref, Opts, E), + {Added2, _Used2, Macs} = import_macros(true, Meta, Ref, Opts, E), {keydelete(Ref, ?key(E, functions)), Macs, Added2}; {only, sigils} -> - {Added1, Funs} = import_sigil_functions(Meta, Ref, Opts, E), - {Added2, Macs} = import_sigil_macros(Meta, Ref, Opts, E), + {Added1, _Used1, Funs} = import_sigil_functions(Meta, Ref, Opts, E), + {Added2, _Used2, Macs} = import_sigil_macros(Meta, Ref, Opts, E), {Funs, Macs, Added1 or Added2}; {only, List} when is_list(List) -> - {Added1, Funs} = import_functions(Meta, Ref, Opts, E), - {Added2, Macs} = import_macros(false, Meta, Ref, Opts, E), + {Added1, Used1, Funs} = import_functions(Meta, Ref, Opts, E), + {Added2, Used2, Macs} = import_macros(false, Meta, Ref, Opts, E), + [elixir_errors:file_warn(Meta, ?key(E, file), ?MODULE, {invalid_import, {Ref, Name, Arity}}) || + {Name, Arity} <- (List -- Used1) -- Used2], {Funs, Macs, Added1 or Added2}; {only, Other} -> elixir_errors:file_error(Meta, E, ?MODULE, {invalid_option, only, Other}); false -> - {Added1, Funs} = import_functions(Meta, Ref, Opts, E), - {Added2, Macs} = import_macros(false, Meta, Ref, Opts, E), + {Added1, _Used1, Funs} = import_functions(Meta, Ref, Opts, E), + {Added2, _Used2, Macs} = import_macros(false, Meta, Ref, Opts, E), {Funs, Macs, Added1 or Added2} end, @@ -95,9 +97,6 @@ calculate(Meta, Key, Opts, Old, File, Existing) -> _ -> elixir_errors:file_error(Meta, File, ?MODULE, only_and_except_given) end, - [elixir_errors:file_warn(Meta, File, ?MODULE, {invalid_import, {Key, Name, Arity}}) || - {Name, Arity} <- Only -- get_exports(Key)], - intersection(Only, Existing()); _ -> @@ -125,17 +124,14 @@ calculate(Meta, Key, Opts, Old, File, Existing) -> %% Normalize the data before storing it case ordsets:from_list(New) of [] -> - {false, keydelete(Key, Old)}; + {false, [], keydelete(Key, Old)}; Set -> ensure_no_special_form_conflict(Meta, File, Key, Set), - {true, [{Key, Set} | keydelete(Key, Old)]} + {true, Set, [{Key, Set} | keydelete(Key, Old)]} end. %% Retrieve functions and macros from modules -get_exports(Module) -> - get_functions(Module) ++ get_macros(Module). - get_functions(Module) -> try Module:'__info__'(functions) @@ -143,14 +139,6 @@ get_functions(Module) -> error:undef -> remove_internals(Module:module_info(exports)) end. -get_macros(Module) -> - case fetch_macros(Module) of - {ok, Macros} -> - Macros; - error -> - [] - end. - fetch_macros(Module) -> try {ok, Module:'__info__'(macros)} From ffa3ef1ccf15f11138cb7af89595e842616e70f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 14 Dec 2023 17:19:42 +0100 Subject: [PATCH 096/185] Remove unused function --- lib/elixir/test/elixir/kernel/parser_test.exs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index b1803d0e511..d5a077632e0 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -1071,11 +1071,6 @@ defmodule Kernel.ParserTest do defp parse!(string), do: Code.string_to_quoted!(string) - defp assert_token_missing(given_messages, string) do - e = assert_raise TokenMissingError, fn -> parse!(string) end - assert_exception_msg(e, given_messages) - end - defp assert_syntax_error(given_messages, source) do e = assert_raise SyntaxError, fn -> parse!(source) end assert_exception_msg(e, given_messages) From b3b14202d95744802b9e6d10fc6308ef35b682bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 17 Dec 2023 11:40:49 +0100 Subject: [PATCH 097/185] Normalize exception handling in diagnostics --- lib/elixir/lib/code.ex | 2 +- lib/elixir/lib/kernel/parallel_compiler.ex | 5 +---- lib/mix/lib/mix/compilers/elixir.ex | 2 +- lib/mix/lib/mix/task.compiler.ex | 2 +- lib/mix/test/mix/tasks/compile_test.exs | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 25ac1ca37b6..b8203d8d7f2 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -212,7 +212,7 @@ defmodule Code do required(:position) => position(), required(:stacktrace) => Exception.stacktrace(), required(:span) => {line :: pos_integer(), column :: pos_integer()} | nil, - optional(:exception) => Exception.t() | nil, + optional(:details) => term(), optional(any()) => any() } diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index 580e60d3482..b616baec0ca 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -901,13 +901,10 @@ defmodule Kernel.ParallelCompiler do severity: :error, stacktrace: stack, span: span, - exception: get_exception(reason) + details: {kind, reason} } end - defp get_exception(exception) when is_exception(exception), do: exception - defp get_exception(_reason), do: nil - defp get_line_span( _file, %{line: line, column: column, end_line: end_line, end_column: end_column}, diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex index 22e2686affc..11167a79bfc 100644 --- a/lib/mix/lib/mix/compilers/elixir.ex +++ b/lib/mix/lib/mix/compilers/elixir.ex @@ -785,7 +785,7 @@ defmodule Mix.Compilers.Elixir do compiler_name: "Elixir", stacktrace: stacktrace, span: span, - details: Map.get(diagnostic, :exception, nil) + details: Map.get(diagnostic, :details, nil) } end diff --git a/lib/mix/lib/mix/task.compiler.ex b/lib/mix/lib/mix/task.compiler.ex index 44055a710f9..5cd4fc989ae 100644 --- a/lib/mix/lib/mix/task.compiler.ex +++ b/lib/mix/lib/mix/task.compiler.ex @@ -47,7 +47,7 @@ defmodule Mix.Task.Compiler do message: IO.chardata(), position: Code.position(), compiler_name: String.t(), - details: Exception.t() | any, + details: term(), stacktrace: Exception.stacktrace(), span: {line :: pos_integer(), column :: pos_integer()} | nil } diff --git a/lib/mix/test/mix/tasks/compile_test.exs b/lib/mix/test/mix/tasks/compile_test.exs index cb73f993a26..4efd0b2eee2 100644 --- a/lib/mix/test/mix/tasks/compile_test.exs +++ b/lib/mix/test/mix/tasks/compile_test.exs @@ -183,7 +183,7 @@ defmodule Mix.Tasks.CompileTest do position: {2, 20}, message: "** (SyntaxError) invalid syntax found on lib/a.ex:2:" <> _, compiler_name: "Elixir", - details: %SyntaxError{} + details: {:error, %SyntaxError{}} } = diagnostic end) end) From 28f8eba9a2ffdc91d192ea949b21b1250220351d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 17 Dec 2023 13:05:48 +0100 Subject: [PATCH 098/185] Improve docs for URI.encode/2 --- lib/elixir/lib/uri.ex | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/elixir/lib/uri.ex b/lib/elixir/lib/uri.ex index c0a02f3a854..fe612a60941 100644 --- a/lib/elixir/lib/uri.ex +++ b/lib/elixir/lib/uri.ex @@ -362,21 +362,23 @@ defmodule URI do end @doc """ - Percent-escapes all characters that require escaping in `string`. + Percent-encodes all characters that require escaping in `string`. - This means reserved characters, such as `:` and `/`, and the - so-called unreserved characters, which have the same meaning both - escaped and unescaped, won't be escaped by default. + By default, this function is meant to escape the whole URI, and + therefore it will escape all characters which are foreign to the + URI specification. Reserved characters (such as `:` and `/`) or + unreserved (such as letters and numbers) are not escaped. - See `encode_www_form/1` if you are interested in escaping reserved - characters too. - - This function also accepts a `predicate` function as an optional + Because different components of a URI require different escaping + rules, this function also accepts a `predicate` function as an optional argument. If passed, this function will be called with each byte in `string` as its argument and should return a truthy value (anything other - than `false` or `nil`) if the given byte should be left as is, or return a - falsy value (`false` or `nil`) if the character should be escaped. Defaults - to `URI.char_unescaped?/1`. + than `false` or `nil`) if the given byte should be left as is, or + return a falsy value (`false` or `nil`) if the character should be + escaped. Defaults to `URI.char_unescaped?/1`. + + See `encode_www_form/1` if you are interested in escaping reserved + characters too. ## Examples From 12fdd8ea9b8f0239411e10007141877e63723a97 Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Thu, 21 Dec 2023 15:46:31 +0100 Subject: [PATCH 099/185] Document the process anti pattern of sending large data (#13194) Follow up to/extension of #13173 --- .../anti-patterns/process-anti-patterns.md | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md index 7a7378279bc..899cbccac04 100644 --- a/lib/elixir/pages/anti-patterns/process-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -188,6 +188,58 @@ iex> Foo.Bucket.get(bucket, "milk") This anti-pattern was formerly known as [Agent obsession](https://github.com/lucasvegi/Elixir-Code-Smells/tree/main#agent-obsession). +## Sending unnecessary data + +#### Problem + +Sending a message to a process can be an expensive operation if the message is big enough. That's because that message will be fully copied to the receiving process, which may be CPU and memory intensive. This is due to Erlang's "share nothing" architecture, where each process has its own memory, which simplifies and speeds up garbage collection. + +This is more obvious when using `send/2`, `GenServer.call/3`, or the initial data in `GenServer.start_link/3`. Notably this also happens when using `spawn/1`, `Task.async/1`, `Task.async_stream/3`, and so on. It is more subtle here as the anonymous function passed to these functions captures the variables it references, and all captured variables will be copied over. By doing this, you can accidentally send way more data to a process than you actually need. + +#### Example + +Imagine you were to implement some simple reporting of IP addresses that made requests against your application. You want to do this asynchronously and not block processing, so you decide to use `spawn/1`. It may seem like a good idea to hand over the whole connection because we might need more data later. However passing the connection results in copying a lot of unnecessary data like the request body, params, etc. + +```elixir +# log_request_ip send the ip to some external service +spawn(fn -> log_request_ip(conn) end) +``` + +This problem also occurs when accessing only the relevant parts: + +```elixir +spawn(fn -> log_request_ip(conn.remote_ip) end) +``` + +This will still copy over all of `conn`, because the `conn` variable is being captured inside the spawned function. The function then extracts the `remote_ip` field, but only after the whole `conn` has been copied over. + +`send/2` and the `GenServer` APIs also rely on message passing. In the example below, the `conn` is once again copied to the underlying `GenServer`: + +```elixir +GenServer.cast(pid, {:report_ip_address, conn}) +``` + +#### Refactoring + +This anti-pattern has many potential remedies: + +* Limit the data you send to the absolute necessary minimum instead of sending an entire struct. For example, don't send an entire `conn` struct if all you need is a couple of fields. +* If the only process that needs data is the one you are sending to, consider making the process fetch that data instead of passing it. +* Some abstractions, such as [`:persistent_term`](https://www.erlang.org/doc/man/persistent_term.html), allows you to share data between processes, as long as such data changes infrequently. + +In our case, limiting the input data is a reasonable strategy. If all we need *right now* is the IP address, then let's only work with that and make sure we're only passing the IP address into the closure, like so: + +```elixir +ip_address = conn.remote_ip +spawn(fn -> log_request_ip(ip_address) end) +``` + +Or in the `GenServer` case: + +```elixir +GenServer.cast(pid, {:report_ip_address, conn.remote_ip}) +``` + ## Unsupervised processes #### Problem From 39ff86c551fe38a01122069628e7d4130a4ff0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 21 Dec 2023 15:47:40 +0100 Subject: [PATCH 100/185] Indent lists --- lib/elixir/pages/anti-patterns/process-anti-patterns.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md index 899cbccac04..c27c2ea0555 100644 --- a/lib/elixir/pages/anti-patterns/process-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -223,9 +223,11 @@ GenServer.cast(pid, {:report_ip_address, conn}) This anti-pattern has many potential remedies: -* Limit the data you send to the absolute necessary minimum instead of sending an entire struct. For example, don't send an entire `conn` struct if all you need is a couple of fields. -* If the only process that needs data is the one you are sending to, consider making the process fetch that data instead of passing it. -* Some abstractions, such as [`:persistent_term`](https://www.erlang.org/doc/man/persistent_term.html), allows you to share data between processes, as long as such data changes infrequently. + * Limit the data you send to the absolute necessary minimum instead of sending an entire struct. For example, don't send an entire `conn` struct if all you need is a couple of fields. + + * If the only process that needs data is the one you are sending to, consider making the process fetch that data instead of passing it. + + * Some abstractions, such as [`:persistent_term`](https://www.erlang.org/doc/man/persistent_term.html), allows you to share data between processes, as long as such data changes infrequently. In our case, limiting the input data is a reasonable strategy. If all we need *right now* is the IP address, then let's only work with that and make sure we're only passing the IP address into the closure, like so: From 4d52d18ef756693a0ba0256fe61e06f92ad61b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 21 Dec 2023 16:17:05 +0100 Subject: [PATCH 101/185] Do not reset column state in tests --- lib/eex/test/eex_test.exs | 57 ++++++++----------- .../elixir/kernel/parallel_compiler_test.exs | 16 ++++-- .../test/elixir/kernel/tracers_test.exs | 2 - 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/lib/eex/test/eex_test.exs b/lib/eex/test/eex_test.exs index 2e60d90354b..8436bd65b3b 100644 --- a/lib/eex/test/eex_test.exs +++ b/lib/eex/test/eex_test.exs @@ -761,38 +761,31 @@ defmodule EExTest do end test "line and column meta" do - parser_options = Code.get_compiler_option(:parser_options) - Code.put_compiler_option(:parser_options, columns: true) - - try do - indentation = 12 - - ast = - EEx.compile_string( - """ - <%= f() %> <% f() %> - <%= f fn -> %> - <%= f() %> - <% end %> - """, - indentation: indentation - ) - - {_, calls} = - Macro.prewalk(ast, [], fn - {:f, meta, _args} = expr, acc -> {expr, [meta | acc]} - other, acc -> {other, acc} - end) - - assert Enum.reverse(calls) == [ - [line: 1, column: indentation + 5], - [line: 1, column: indentation + 15], - [line: 2, column: indentation + 7], - [line: 3, column: indentation + 9] - ] - after - Code.put_compiler_option(:parser_options, parser_options) - end + indentation = 12 + + ast = + EEx.compile_string( + """ + <%= f() %> <% f() %> + <%= f fn -> %> + <%= f() %> + <% end %> + """, + indentation: indentation + ) + + {_, calls} = + Macro.prewalk(ast, [], fn + {:f, meta, _args} = expr, acc -> {expr, [meta | acc]} + other, acc -> {other, acc} + end) + + assert Enum.reverse(calls) == [ + [line: 1, column: indentation + 5], + [line: 1, column: indentation + 15], + [line: 2, column: indentation + 7], + [line: 3, column: indentation + 9] + ] end end diff --git a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs index 21de7d48132..c81e2014578 100644 --- a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs +++ b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs @@ -147,7 +147,7 @@ defmodule Kernel.ParallelCompilerTest do expected_msg = "Undef.__struct__/1 is undefined, cannot expand struct Undef" assert capture_io(:stderr, fn -> - assert {:error, [{^fixture, 3, msg}, {^fixture, 0, compile_msg}], []} = + assert {:error, [{^fixture, {3, 5}, msg}, {^fixture, 0, compile_msg}], []} = Kernel.ParallelCompiler.compile([fixture]) assert msg =~ expected_msg @@ -216,7 +216,7 @@ defmodule Kernel.ParallelCompilerTest do "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" assert capture_io(:stderr, fn -> - assert {:error, [{^fixture, 7, msg}, {^fixture, 0, compile_msg}], []} = + assert {:error, [{^fixture, {7, 3}, msg}, {^fixture, 0, compile_msg}], []} = Kernel.ParallelCompiler.compile([fixture]) assert msg =~ expected_msg @@ -245,7 +245,9 @@ defmodule Kernel.ParallelCompilerTest do "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" assert capture_io(:stderr, fn -> - assert {:error, [{^missing_struct, 2, msg}, {^missing_struct, 0, compile_msg}], []} = + assert {:error, + [{^missing_struct, {2, 3}, msg}, {^missing_struct, 0, compile_msg}], + []} = Kernel.ParallelCompiler.compile([missing_struct, depends_on]) assert msg =~ expected_msg @@ -272,7 +274,9 @@ defmodule Kernel.ParallelCompilerTest do expected_msg = "module Unknown.Module is not loaded and could not be found" assert capture_io(:stderr, fn -> - assert {:error, [{^missing_import, 2, msg}, {^missing_import, 0, compile_msg}], []} = + assert {:error, + [{^missing_import, {2, 3}, msg}, {^missing_import, 0, compile_msg}], + []} = Kernel.ParallelCompiler.compile([missing_import, depends_on]) assert msg =~ expected_msg @@ -509,7 +513,7 @@ defmodule Kernel.ParallelCompilerTest do expected_msg = "Undef.__struct__/1 is undefined, cannot expand struct Undef" assert capture_io(:stderr, fn -> - assert {:error, [{^fixture, 3, msg}, {^fixture, 0, compile_msg}], []} = + assert {:error, [{^fixture, {3, 5}, msg}, {^fixture, 0, compile_msg}], []} = Kernel.ParallelCompiler.require([fixture]) assert msg =~ expected_msg @@ -537,7 +541,7 @@ defmodule Kernel.ParallelCompilerTest do "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" assert capture_io(:stderr, fn -> - assert {:error, [{^fixture, 7, msg}, {^fixture, 0, compile_msg}], []} = + assert {:error, [{^fixture, {7, 3}, msg}, {^fixture, 0, compile_msg}], []} = Kernel.ParallelCompiler.require([fixture]) assert msg =~ expected_msg diff --git a/lib/elixir/test/elixir/kernel/tracers_test.exs b/lib/elixir/test/elixir/kernel/tracers_test.exs index 6bc2ccf6592..08826bf324a 100644 --- a/lib/elixir/test/elixir/kernel/tracers_test.exs +++ b/lib/elixir/test/elixir/kernel/tracers_test.exs @@ -16,11 +16,9 @@ defmodule Kernel.TracersTest do setup_all do Code.put_compiler_option(:tracers, [__MODULE__]) - Code.put_compiler_option(:parser_options, columns: true) on_exit(fn -> Code.put_compiler_option(:tracers, []) - Code.put_compiler_option(:parser_options, []) end) end From 7db4413476ba584d5822fc2b0d268496ae345683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 21 Dec 2023 17:05:47 +0100 Subject: [PATCH 102/185] Add source field to diagnostics --- lib/elixir/lib/code.ex | 7 +++ lib/elixir/lib/kernel/parallel_compiler.ex | 50 ++++++++++--------- lib/elixir/lib/module/parallel_checker.ex | 1 + lib/elixir/src/elixir_errors.erl | 1 + lib/elixir/test/elixir/code_test.exs | 1 + .../elixir/kernel/parallel_compiler_test.exs | 18 +++++++ lib/mix/lib/mix/compilers/elixir.ex | 6 ++- lib/mix/lib/mix/compilers/erlang.ex | 5 +- lib/mix/lib/mix/task.compiler.ex | 8 +++ .../test/mix/tasks/compile.elixir_test.exs | 4 ++ .../test/mix/tasks/compile.erlang_test.exs | 2 + lib/mix/test/mix/tasks/compile.leex_test.exs | 1 + lib/mix/test/mix/tasks/compile.yecc_test.exs | 3 ++ lib/mix/test/mix/tasks/compile_test.exs | 3 ++ 14 files changed, 85 insertions(+), 25 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index b8203d8d7f2..de1a9fa9c87 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -197,6 +197,7 @@ defmodule Code do @typedoc """ Diagnostics returned by the compiler and code evaluation. + The file and position relate to where the diagnostic should be shown. If there is a file and position, then the diagnostic is precise and you can use the given file and position for generating snippets, IDEs annotations, and so on. An optional span is available with @@ -204,8 +205,14 @@ defmodule Code do Otherwise, a stacktrace may be given, which you can place your own heuristics to provide better reporting. + + The source field points to the source file the compiler tracked + the error to. For example, a file `lib/foo.ex` may embed `.eex` + templates from `lib/foo/bar.eex`. A syntax error on the EEx template + will point to file `lib/foo/bar.eex` but the source is `lib/foo.ex`. """ @type diagnostic(severity) :: %{ + required(:source) => Path.t() | nil, required(:file) => Path.t() | nil, required(:severity) => severity, required(:message) => String.t(), diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index b616baec0ca..d88b23dc730 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -859,9 +859,12 @@ defmodule Kernel.ParallelCompiler do ) for {file, _, description, stacktrace} <- deadlock do + file = Path.absname(file) + %{ severity: :error, - file: Path.absname(file), + file: file, + source: file, position: nil, message: description, stacktrace: stacktrace, @@ -889,13 +892,14 @@ defmodule Kernel.ParallelCompiler do ]) end - defp to_error(file, kind, reason, stack) do - {line, span} = get_line_span(file, reason, stack) - file = Path.absname(file) + defp to_error(source, kind, reason, stack) do + {file, line, span} = get_snippet_info(source, reason, stack) + source = Path.absname(source) message = :unicode.characters_to_binary(Kernel.CLI.format_error(kind, reason, stack)) %{ - file: file, + file: file || source, + source: source, position: line || 0, message: message, severity: :error, @@ -905,47 +909,47 @@ defmodule Kernel.ParallelCompiler do } end - defp get_line_span( + defp get_snippet_info( _file, - %{line: line, column: column, end_line: end_line, end_column: end_column}, + %{file: file, line: line, column: column, end_line: end_line, end_column: end_column}, _stack ) when is_integer(line) and line > 0 and is_integer(column) and column >= 0 and is_integer(end_line) and end_line > 0 and is_integer(end_column) and end_column >= 0 do - {{line, column}, {end_line, end_column}} + {Path.absname(file), {line, column}, {end_line, end_column}} end - defp get_line_span(_file, %{line: line, column: column}, _stack) + defp get_snippet_info(_file, %{file: file, line: line, column: column}, _stack) when is_integer(line) and line > 0 and is_integer(column) and column >= 0 do - {{line, column}, nil} + {Path.absname(file), {line, column}, nil} end - defp get_line_span(_file, %{line: line}, _stack) when is_integer(line) and line > 0 do - {line, nil} + defp get_snippet_info(_file, %{line: line}, _stack) when is_integer(line) and line > 0 do + {nil, line, nil} end - defp get_line_span(file, :undef, [{_, _, _, []}, {_, _, _, info} | _]) do - get_line_span_from_stacktrace_info(info, file) + defp get_snippet_info(file, :undef, [{_, _, _, []}, {_, _, _, info} | _]) do + get_snippet_info_from_stacktrace_info(info, file) end - defp get_line_span(file, _reason, [{_, _, _, [file: expanding]}, {_, _, _, info} | _]) + defp get_snippet_info(file, _reason, [{_, _, _, [file: expanding]}, {_, _, _, info} | _]) when expanding in [~c"expanding macro", ~c"expanding struct"] do - get_line_span_from_stacktrace_info(info, file) + get_snippet_info_from_stacktrace_info(info, file) end - defp get_line_span(file, _reason, [{_, _, _, info} | _]) do - get_line_span_from_stacktrace_info(info, file) + defp get_snippet_info(file, _reason, [{_, _, _, info} | _]) do + get_snippet_info_from_stacktrace_info(info, file) end - defp get_line_span(_, _, _) do - {nil, nil} + defp get_snippet_info(_, _, _) do + {nil, nil, nil} end - defp get_line_span_from_stacktrace_info(info, file) do + defp get_snippet_info_from_stacktrace_info(info, file) do if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do - {Keyword.get(info, :line), nil} + {nil, Keyword.get(info, :line), nil} else - {nil, nil} + {nil, nil, nil} end end end diff --git a/lib/elixir/lib/module/parallel_checker.ex b/lib/elixir/lib/module/parallel_checker.ex index 1dab57db3d7..d4e51abb9e5 100644 --- a/lib/elixir/lib/module/parallel_checker.ex +++ b/lib/elixir/lib/module/parallel_checker.ex @@ -324,6 +324,7 @@ defmodule Module.ParallelChecker do defp to_diagnostic(message, {file, position, mfa}) when is_list(position) do %{ severity: :warning, + source: file, file: file, position: position_to_tuple(position), message: IO.iodata_to_binary(message), diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index cb0e3a17600..d82b15ed867 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -106,6 +106,7 @@ emit_diagnostic(Severity, Position, File, Message, Stacktrace, Options) -> Diagnostic = #{ severity => Severity, + source => File, file => File, position => Position, message => unicode:characters_to_binary(Message), diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index de6733d99d3..8d4dcb1a751 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -252,6 +252,7 @@ defmodule CodeTest do message: "undefined variable \"x\"", position: 1, file: "nofile", + source: "nofile", stacktrace: [], severity: :error } diff --git a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs index c81e2014578..9edbe5c0fc1 100644 --- a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs +++ b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs @@ -480,6 +480,24 @@ defmodule Kernel.ParallelCompilerTest do end) end + test "gets correct file+line+column number for SyntaxError" do + File.mkdir_p!(tmp_path()) + + [fixture] = + write_tmp("error", + error: """ + raise SyntaxError, file: "foo/bar.ex", line: 3, column: 10 + """ + ) + + file = Path.absname("foo/bar.ex") + + capture_io(:stderr, fn -> + assert {:error, [%{file: ^file, source: ^fixture, position: {3, 10}}], _} = + Kernel.ParallelCompiler.compile([fixture], return_diagnostics: true) + end) + end + test "gets proper beam destinations from dynamic modules" do fixtures = write_tmp( diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex index 11167a79bfc..937d7eba1af 100644 --- a/lib/mix/lib/mix/compilers/elixir.ex +++ b/lib/mix/lib/mix/compilers/elixir.ex @@ -724,9 +724,11 @@ defmodule Mix.Compilers.Elixir do )} <- sources, file = Path.absname(source), {position, message, span} <- compile_warnings ++ runtime_warnings do + # TODO: Store the whole diagnostic diagnostic = %Mix.Task.Compiler.Diagnostic{ severity: :warning, file: file, + source: file, position: position, message: message, compiler_name: "Elixir", @@ -774,11 +776,13 @@ defmodule Mix.Compilers.Elixir do message: message, severity: severity, stacktrace: stacktrace, - span: span + span: span, + source: source } = diagnostic ) do %Mix.Task.Compiler.Diagnostic{ file: file, + source: source, position: position, message: message, severity: severity, diff --git a/lib/mix/lib/mix/compilers/erlang.ex b/lib/mix/lib/mix/compilers/erlang.ex index 7cc2fa58c19..b97171df38a 100644 --- a/lib/mix/lib/mix/compilers/erlang.ex +++ b/lib/mix/lib/mix/compilers/erlang.ex @@ -288,8 +288,11 @@ defmodule Mix.Compilers.Erlang do defp to_diagnostics(warnings_or_errors, severity) do for {file, issues} <- warnings_or_errors, {location, module, data} <- issues do + file = Path.absname(file) + %Mix.Task.Compiler.Diagnostic{ - file: Path.absname(file), + file: file, + source: file, position: location_normalize(location), message: to_string(module.format_error(data)), severity: severity, diff --git a/lib/mix/lib/mix/task.compiler.ex b/lib/mix/lib/mix/task.compiler.ex index 5cd4fc989ae..cd0856d4202 100644 --- a/lib/mix/lib/mix/task.compiler.ex +++ b/lib/mix/lib/mix/task.compiler.ex @@ -32,6 +32,7 @@ defmodule Mix.Task.Compiler do @moduledoc """ Diagnostic information such as a warning or compilation error. + The file and position relate to where the diagnostic should be shown. If there is a file and position, then the diagnostic is precise and you can use the given file and position for generating snippets, IDEs annotations, and so on. An optional span is available with @@ -39,10 +40,16 @@ defmodule Mix.Task.Compiler do Otherwise, a stacktrace may be given, which you can place your own heuristics to provide better reporting. + + The source field points to the source file the compiler tracked + the error to. For example, a file `lib/foo.ex` may embed `.eex` + templates from `lib/foo/bar.eex`. A syntax error on the EEx template + will point to file `lib/foo/bar.eex` but the source is `lib/foo.ex`. """ @type t :: %__MODULE__{ file: Path.t() | nil, + source: Path.t() | nil, severity: severity, message: IO.chardata(), position: Code.position(), @@ -72,6 +79,7 @@ defmodule Mix.Task.Compiler do @enforce_keys [:file, :severity, :message, :position, :compiler_name] defstruct [ :file, + :source, :severity, :message, :position, diff --git a/lib/mix/test/mix/tasks/compile.elixir_test.exs b/lib/mix/test/mix/tasks/compile.elixir_test.exs index 9666cbf5db4..506b29f9e1f 100644 --- a/lib/mix/test/mix/tasks/compile.elixir_test.exs +++ b/lib/mix/test/mix/tasks/compile.elixir_test.exs @@ -1443,6 +1443,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do assert %Diagnostic{ file: ^file, + source: ^file, severity: :warning, position: {2, 13}, compiler_name: "Elixir", @@ -1457,6 +1458,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do assert %Diagnostic{ file: ^file, + source: ^file, severity: :warning, position: {2, 13}, compiler_name: "Elixir", @@ -1518,6 +1520,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do assert %Diagnostic{ file: ^file, + source: ^file, severity: :error, position: {2, 20}, message: "** (SyntaxError) invalid syntax found on lib/a.ex:2:" <> _, @@ -1552,6 +1555,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do assert %Diagnostic{ file: ^file, + source: ^file, severity: :error, position: 2, message: "** (KeyError) key :invalid_key not found" <> _, diff --git a/lib/mix/test/mix/tasks/compile.erlang_test.exs b/lib/mix/test/mix/tasks/compile.erlang_test.exs index d76f1cbcca6..a3440ea6384 100644 --- a/lib/mix/test/mix/tasks/compile.erlang_test.exs +++ b/lib/mix/test/mix/tasks/compile.erlang_test.exs @@ -95,6 +95,7 @@ defmodule Mix.Tasks.Compile.ErlangTest do assert %Mix.Task.Compiler.Diagnostic{ compiler_name: "erl_parse", file: ^file, + source: ^file, message: "syntax error before: zzz", position: position(2, 5), severity: :error @@ -120,6 +121,7 @@ defmodule Mix.Tasks.Compile.ErlangTest do assert %Mix.Task.Compiler.Diagnostic{ file: ^file, + source: ^file, compiler_name: "erl_lint", message: "function my_fn/0 is unused", position: position(2, 1), diff --git a/lib/mix/test/mix/tasks/compile.leex_test.exs b/lib/mix/test/mix/tasks/compile.leex_test.exs index 25c82b8d293..cbc5378b209 100644 --- a/lib/mix/test/mix/tasks/compile.leex_test.exs +++ b/lib/mix/test/mix/tasks/compile.leex_test.exs @@ -24,6 +24,7 @@ defmodule Mix.Tasks.Compile.LeexTest do assert %Mix.Task.Compiler.Diagnostic{ compiler_name: "leex", file: ^file, + source: ^file, message: "missing Definitions", position: 1, severity: :error diff --git a/lib/mix/test/mix/tasks/compile.yecc_test.exs b/lib/mix/test/mix/tasks/compile.yecc_test.exs index faac3b9600f..0eb4cf56757 100644 --- a/lib/mix/test/mix/tasks/compile.yecc_test.exs +++ b/lib/mix/test/mix/tasks/compile.yecc_test.exs @@ -26,6 +26,7 @@ defmodule Mix.Tasks.Compile.YeccTest do assert %Mix.Task.Compiler.Diagnostic{ compiler_name: "yecc", file: ^file, + source: ^file, message: message, position: position(1, 5), severity: :error @@ -56,6 +57,7 @@ defmodule Mix.Tasks.Compile.YeccTest do assert %Mix.Task.Compiler.Diagnostic{ compiler_name: "yecc", file: ^file, + source: ^file, message: "conflicts: 1 shift/reduce, 0 reduce/reduce", position: 0, severity: :warning @@ -73,6 +75,7 @@ defmodule Mix.Tasks.Compile.YeccTest do assert %Mix.Task.Compiler.Diagnostic{ compiler_name: "yecc", file: ^file, + source: ^file, message: "conflicts: 1 shift/reduce, 0 reduce/reduce", position: 0, severity: :warning diff --git a/lib/mix/test/mix/tasks/compile_test.exs b/lib/mix/test/mix/tasks/compile_test.exs index 4efd0b2eee2..a812a9bd812 100644 --- a/lib/mix/test/mix/tasks/compile_test.exs +++ b/lib/mix/test/mix/tasks/compile_test.exs @@ -179,6 +179,7 @@ defmodule Mix.Tasks.CompileTest do assert %Mix.Task.Compiler.Diagnostic{ file: ^file, + source: ^file, severity: :error, position: {2, 20}, message: "** (SyntaxError) invalid syntax found on lib/a.ex:2:" <> _, @@ -213,6 +214,7 @@ defmodule Mix.Tasks.CompileTest do assert %Mix.Task.Compiler.Diagnostic{ file: ^file, + source: ^file, severity: :error, position: 3, message: "** (RuntimeError) error\n expanding macro: A.custom_macro/0" <> _, @@ -242,6 +244,7 @@ defmodule Mix.Tasks.CompileTest do assert %Mix.Task.Compiler.Diagnostic{ compiler_name: "erl_parse", file: ^file, + source: ^file, message: "syntax error before: b", position: position(2, 5), severity: :error From 0ae4bfb2225a09e559fa7b8e6fec40f5e19747e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 21 Dec 2023 17:22:05 +0100 Subject: [PATCH 103/185] Preserve diagnostics based on source field, closes #13142 --- lib/mix/lib/mix/compilers/elixir.ex | 41 +++++++++-------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex index 937d7eba1af..eb9da80e21c 100644 --- a/lib/mix/lib/mix/compilers/elixir.ex +++ b/lib/mix/lib/mix/compilers/elixir.ex @@ -1,7 +1,7 @@ defmodule Mix.Compilers.Elixir do @moduledoc false - @manifest_vsn 22 + @manifest_vsn 23 @checkpoint_vsn 2 import Record @@ -171,7 +171,7 @@ defmodule Mix.Compilers.Elixir do state = {%{}, exports, sources, [], modules, removed_modules} compiler_loop(stale, stale_modules, dest, timestamp, opts, state) else - {:ok, info, state} -> + {:ok, %{runtime_warnings: runtime_warnings, compile_warnings: compile_warnings}, state} -> {modules, _exports, sources, _changed, pending_modules, _stale_exports} = state previous_warnings = @@ -179,7 +179,9 @@ defmodule Mix.Compilers.Elixir do do: previous_warnings(sources, true), else: [] - sources = apply_warnings(sources, info) + runtime_warnings = Enum.map(runtime_warnings, &diagnostic/1) + compile_warnings = Enum.map(compile_warnings, &diagnostic/1) + sources = apply_warnings(sources, runtime_warnings, compile_warnings) write_manifest( manifest, @@ -193,8 +195,7 @@ defmodule Mix.Compilers.Elixir do ) put_compile_env(sources) - info_warnings = info.runtime_warnings ++ info.compile_warnings - all_warnings = previous_warnings ++ Enum.map(info_warnings, &diagnostic/1) + all_warnings = previous_warnings ++ runtime_warnings ++ compile_warnings unless_previous_warnings_as_errors(previous_warnings, opts, {:ok, all_warnings}) {:error, errors, %{runtime_warnings: r_warnings, compile_warnings: c_warnings}, state} -> @@ -717,25 +718,9 @@ defmodule Mix.Compilers.Elixir do end defp previous_warnings(sources, print?) do - for {source, - source( - compile_warnings: compile_warnings, - runtime_warnings: runtime_warnings - )} <- sources, - file = Path.absname(source), - {position, message, span} <- compile_warnings ++ runtime_warnings do - # TODO: Store the whole diagnostic - diagnostic = %Mix.Task.Compiler.Diagnostic{ - severity: :warning, - file: file, - source: file, - position: position, - message: message, - compiler_name: "Elixir", - stacktrace: [], - span: span - } - + for {_, source(compile_warnings: compile_warnings, runtime_warnings: runtime_warnings)} <- + sources, + diagnostic <- compile_warnings ++ runtime_warnings do if print? do Mix.shell().print_app() Code.print_diagnostic(diagnostic) @@ -745,13 +730,13 @@ defmodule Mix.Compilers.Elixir do end end - defp apply_warnings(sources, %{runtime_warnings: [], compile_warnings: []}) do + defp apply_warnings(sources, [], []) do sources end - defp apply_warnings(sources, %{runtime_warnings: r_warnings, compile_warnings: c_warnings}) do - runtime_group = Enum.group_by(r_warnings, & &1.file, &{&1.position, &1.message, &1.span}) - compile_group = Enum.group_by(c_warnings, & &1.file, &{&1.position, &1.message, &1.span}) + defp apply_warnings(sources, runtime_warnings, compile_warnings) do + runtime_group = Enum.group_by(runtime_warnings, & &1.source) + compile_group = Enum.group_by(compile_warnings, & &1.source) for {source_path, source_entry} <- sources, into: %{} do key = Path.absname(source_path) From dfafea092ea936d1d23db2df4f12821957c6e5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 21 Dec 2023 19:26:06 +0100 Subject: [PATCH 104/185] Consider column in snippets, closes #13199 --- lib/elixir/src/elixir.hrl | 1 + lib/elixir/src/elixir_errors.erl | 30 ++++++++++++------- lib/elixir/src/elixir_interpolation.erl | 2 +- lib/elixir/src/elixir_tokenizer.erl | 14 ++++----- .../test/elixir/kernel/diagnostics_test.exs | 29 +++++++++--------- 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/lib/elixir/src/elixir.hrl b/lib/elixir/src/elixir.hrl index e5d12c93386..7aa95222988 100644 --- a/lib/elixir/src/elixir.hrl +++ b/lib/elixir/src/elixir.hrl @@ -35,6 +35,7 @@ identifier_tokenizer=elixir_tokenizer, ascii_identifiers_only=true, indentation=0, + column=1, mismatch_hints=[], warnings=[] }). diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index d82b15ed867..96d9ea6ec5b 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -321,7 +321,7 @@ parse_error(Location, File, Error, <<>>, Input) -> _ -> <> end, case lists:keytake(error_type, 1, Location) of - {value, {error_type, unclosed_delimiter}, Loc} -> raise_token_missing(Loc, File, Input, Message); + {value, {error_type, unclosed_delimiter}, Loc} -> raise_unclosed_delimiter(Loc, File, Input, Message); _ -> raise_snippet(Location, File, Input, 'Elixir.TokenMissingError', Message) end; @@ -408,15 +408,16 @@ parse_erl_term(Term) -> Parsed. raise_mismatched_delimiter(Location, File, Input, Message) -> - {InputString, StartLine, _} = Input, - InputBinary = elixir_utils:characters_to_binary(InputString), - KV = [{file, File}, {line_offset, StartLine - 1}, {snippet, InputBinary} | Location], - raise('Elixir.MismatchedDelimiterError', Message, KV). + {InputString, StartLine, StartColumn} = Input, + Snippet = indent(elixir_utils:characters_to_binary(InputString), StartColumn), + Opts = [{file, File}, {line_offset, StartLine - 1}, {snippet, Snippet} | Location], + raise('Elixir.MismatchedDelimiterError', Message, Opts). -raise_token_missing(Location, File, Input, Message) -> - {InputString, StartLine, _} = Input, - InputBinary = elixir_utils:characters_to_binary(InputString), - raise('Elixir.TokenMissingError', Message, [{line_offset, StartLine - 1}, {file, File}, {snippet, InputBinary} | Location]). +raise_unclosed_delimiter(Location, File, Input, Message) -> + {InputString, StartLine, StartColumn} = Input, + Snippet = indent(elixir_utils:characters_to_binary(InputString), StartColumn), + Opts = [{line_offset, StartLine - 1}, {file, File}, {snippet, Snippet} | Location], + raise('Elixir.TokenMissingError', Message, Opts). raise_reserved(Location, File, Input, Keyword) -> raise_snippet(Location, File, Input, 'Elixir.SyntaxError', @@ -425,9 +426,16 @@ raise_reserved(Location, File, Input, Keyword) -> "it can't be used as a variable or be defined nor invoked as a regular function">>). raise_snippet(Location, File, Input, Kind, Message) when is_binary(File) -> - {InputString, StartLine, _} = Input, + {InputString, StartLine, StartColumn} = Input, Snippet = snippet_line(InputString, Location, StartLine), - raise(Kind, Message, [{file, File}, {snippet, Snippet} | Location]). + raise(Kind, Message, [{file, File}, {snippet, indent(Snippet, StartColumn)} | Location]). + +indent(nil, _StartColumn) -> nil; +indent(Snippet, StartColumn) when StartColumn > 1 -> + Prefix = binary:copy(<<" ">>, StartColumn - 1), + Replaced = binary:replace(Snippet, <<"\n">>, <>, [global]), + <>; +indent(Snippet, _StartColumn) -> Snippet. snippet_line(InputString, Location, StartLine) -> {line, Line} = lists:keyfind(line, 1, Location), diff --git a/lib/elixir/src/elixir_interpolation.erl b/lib/elixir/src/elixir_interpolation.erl index c63370dfe30..d4ccf3086cc 100644 --- a/lib/elixir/src/elixir_interpolation.erl +++ b/lib/elixir/src/elixir_interpolation.erl @@ -97,7 +97,7 @@ extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, [H,H,H] = Last) -> extract(NewRest, NewBuffer, Output, Line + 1, Column, Scope, Interpol, Last) end; extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, Last) -> - extract(Rest, Buffer, Output, Line + 1, 1, Scope, Interpol, Last). + extract(Rest, Buffer, Output, Line + 1, Scope#elixir_tokenizer.column, Scope, Interpol, Last). strip_horizontal_space([H | T], Buffer, Counter) when H =:= $\s; H =:= $\t -> strip_horizontal_space(T, [H | Buffer], Counter + 1); diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 54c3dcc19d9..03697a1953e 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -125,7 +125,7 @@ tokenize(String, Line, Column, Opts) -> Acc#elixir_tokenizer{unescape=Unescape}; (_, Acc) -> Acc - end, #elixir_tokenizer{identifier_tokenizer=IdentifierTokenizer}, Opts), + end, #elixir_tokenizer{identifier_tokenizer=IdentifierTokenizer, column=Column}, Opts), tokenize(String, Line, Column, Scope, []). @@ -712,9 +712,9 @@ unexpected_token([T | Rest], Line, Column, Scope, Tokens) -> error({?LOC(Line, Column), "unexpected token: ", Message}, Rest, Scope, Tokens). tokenize_eol(Rest, Line, Scope, Tokens) -> - {StrippedRest, Indentation} = strip_horizontal_space(Rest, 0), - IndentedScope = Scope#elixir_tokenizer{indentation=Indentation}, - tokenize(StrippedRest, Line + 1, Indentation + 1, IndentedScope, Tokens). + {StrippedRest, Column} = strip_horizontal_space(Rest, Scope#elixir_tokenizer.column), + IndentedScope = Scope#elixir_tokenizer{indentation=Column-1}, + tokenize(StrippedRest, Line + 1, Column, IndentedScope, Tokens). strip_horizontal_space([H | T], Counter) when ?is_horizontal_space(H) -> strip_horizontal_space(T, Counter + 1); @@ -730,12 +730,12 @@ tokenize_dot(T, Line, Column, DotInfo, Scope, Tokens) -> {Rest, Comment} -> preserve_comments(Line, Column, Tokens, Comment, Rest, Scope), - tokenize_dot(Rest, Line, 1, DotInfo, Scope, Tokens) + tokenize_dot(Rest, Line, Scope#elixir_tokenizer.column, DotInfo, Scope, Tokens) end; {"\r\n" ++ Rest, _} -> - tokenize_dot(Rest, Line + 1, 1, DotInfo, Scope, Tokens); + tokenize_dot(Rest, Line + 1, Scope#elixir_tokenizer.column, DotInfo, Scope, Tokens); {"\n" ++ Rest, _} -> - tokenize_dot(Rest, Line + 1, 1, DotInfo, Scope, Tokens); + tokenize_dot(Rest, Line + 1, Scope#elixir_tokenizer.column, DotInfo, Scope, Tokens); {Rest, Length} -> handle_dot([$. | Rest], Line, Column + Length, DotInfo, Scope, Tokens) end. diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs index 4a057180b9b..d757a41f1d5 100644 --- a/lib/elixir/test/elixir/kernel/diagnostics_test.exs +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -638,24 +638,25 @@ defmodule Kernel.DiagnosticsTest do """ end - test "TokenMissingError (snippet) with offset" do + test "TokenMissingError (snippet) with offset and column" do output = capture_raise( """ 1 + """, TokenMissingError, - line: 3 + line: 3, + column: 3 ) assert output == """ - ** (TokenMissingError) token missing on nofile:3:4: + ** (TokenMissingError) token missing on nofile:3:6: error: syntax error: expression is incomplete │ - 3 │ 1 + - │ ^ + 3 │ 1 + + │ ^ │ - └─ nofile:3:4\ + └─ nofile:3:6\ """ end @@ -852,7 +853,7 @@ defmodule Kernel.DiagnosticsTest do └─ #{path}:3: Sample.a/0 """ - assert capture_eval(source, false) =~ expected + assert capture_eval(source, columns: false) =~ expected after purge(Sample) end @@ -1086,7 +1087,7 @@ defmodule Kernel.DiagnosticsTest do """ - assert capture_eval(source, false) == expected + assert capture_eval(source, columns: false) == expected after purge(Sample) end @@ -1117,7 +1118,7 @@ defmodule Kernel.DiagnosticsTest do """ - assert capture_compile(source, false) == expected + assert capture_compile(source, columns: false) == expected after purge(Sample) end @@ -1290,7 +1291,7 @@ defmodule Kernel.DiagnosticsTest do """ - assert capture_eval(source, false) == expected + assert capture_eval(source, columns: false) == expected after purge(Sample) end @@ -1443,17 +1444,17 @@ defmodule Kernel.DiagnosticsTest do |> Path.relative_to_cwd() end - defp capture_eval(source, columns? \\ true) do + defp capture_eval(source, opts \\ [columns: true]) do capture_io(:stderr, fn -> - quoted = Code.string_to_quoted!(source, columns: columns?) + quoted = Code.string_to_quoted!(source, opts) Code.eval_quoted(quoted) end) end - defp capture_compile(source, columns? \\ true) do + defp capture_compile(source, opts \\ [columns: true]) do capture_io(:stderr, fn -> assert_raise CompileError, fn -> - ast = Code.string_to_quoted!(source, columns: columns?) + ast = Code.string_to_quoted!(source, opts) Code.eval_quoted(ast) end end) From 1cba584ac811b48a03f0d608770832b732ac8197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 21 Dec 2023 19:48:39 +0100 Subject: [PATCH 105/185] Fix column precision in test --- lib/elixir/test/elixir/code_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index 8d4dcb1a751..7db20b4f32c 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -484,7 +484,7 @@ defmodule CodeTest do assert_exception( SyntaxError, - ["nofile:11:5:", "syntax error before:", "1 + * 3", "^"], + ["nofile:11:7:", "syntax error before:", "1 + * 3", "^"], fn -> Code.string_to_quoted!(":ok\n1 + * 3", line: 10, column: 3) end From 5bd5e75c3bac734944b7bc232d42bdfabb64374e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 21 Dec 2023 21:14:44 +0100 Subject: [PATCH 106/185] Extract snippet in elixir_errors to simplify exception handling --- lib/elixir/lib/exception.ex | 64 +++++----------------- lib/elixir/src/elixir_errors.erl | 83 ++++++++++++++--------------- lib/elixir/src/elixir_tokenizer.erl | 2 - 3 files changed, 54 insertions(+), 95 deletions(-) diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index bbdeefee3d6..fe0f45ba018 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -805,7 +805,6 @@ defmodule Exception do def format_snippet( {start_line, _start_column} = start_pos, {end_line, end_column} = end_pos, - line_offset, description, file, lines, @@ -819,22 +818,12 @@ defmodule Exception do relevant_lines = if end_line - start_line < 5 do - line_range( - lines, - start_pos, - end_pos, - line_offset, - padding, - max_digits, - start_message, - end_message - ) + line_range(lines, start_pos, end_pos, padding, max_digits, start_message, end_message) else trimmed_inbetween_lines( lines, start_pos, end_pos, - line_offset, padding, max_digits, start_message, @@ -854,7 +843,6 @@ defmodule Exception do def format_snippet( {start_line, start_column}, {end_line, end_column}, - line_offset, description, file, lines, @@ -865,9 +853,7 @@ defmodule Exception do max_digits = digits(end_line) general_padding = max(2, max_digits) + 1 padding = n_spaces(general_padding) - - line = Enum.fetch!(lines, end_line - 1 - line_offset) - formatted_line = [line_padding(end_line, max_digits), to_string(end_line), " │ ", line] + formatted_line = [line_padding(end_line, max_digits), to_string(end_line), " │ ", hd(lines)] mismatched_closing_line = [ @@ -914,7 +900,6 @@ defmodule Exception do lines, {start_line, start_column}, {end_line, end_column}, - line_offset, padding, max_digits, start_message, @@ -922,8 +907,8 @@ defmodule Exception do ) do start_padding = line_padding(start_line, max_digits) end_padding = line_padding(end_line, max_digits) - first_line = Enum.fetch!(lines, start_line - 1 - line_offset) - last_line = Enum.fetch!(lines, end_line - 1 - line_offset) + first_line = hd(lines) + last_line = List.last(lines) """ #{start_padding}#{start_line} │ #{first_line} @@ -938,22 +923,12 @@ defmodule Exception do lines, {start_line, start_column}, {end_line, end_column}, - line_offset, padding, max_digits, start_message, end_message ) do - start_line = start_line - 1 - end_line = end_line - 1 - - lines - |> Enum.slice((start_line - line_offset)..(end_line - line_offset)) - |> Enum.zip_with(start_line..end_line, fn line, line_number -> - line_number = line_number + 1 - start_line = start_line + 1 - end_line = end_line + 1 - + Enum.zip_with(lines, start_line..end_line, fn line, line_number -> line_padding = line_padding(line_number, max_digits) cond do @@ -1180,7 +1155,6 @@ defmodule MismatchedDelimiterError do :file, :line, :column, - :line_offset, :end_line, :end_column, :opening_delimiter, @@ -1196,7 +1170,6 @@ defmodule MismatchedDelimiterError do column: start_column, end_line: end_line, end_column: end_column, - line_offset: line_offset, description: description, expected_delimiter: expected_delimiter, file: file, @@ -1214,7 +1187,6 @@ defmodule MismatchedDelimiterError do Exception.format_snippet( start_pos, end_pos, - line_offset, description, file, lines, @@ -1305,7 +1277,6 @@ defmodule TokenMissingError do :column, :end_line, :end_column, - :line_offset, :snippet, :opening_delimiter, :expected_delimiter, @@ -1318,20 +1289,19 @@ defmodule TokenMissingError do line: line, column: column, end_line: end_line, - line_offset: line_offset, description: description, expected_delimiter: expected_delimiter, snippet: snippet }) when not is_nil(snippet) and not is_nil(column) and not is_nil(end_line) do - {lines, total_trimmed_lines} = handle_trailing_newlines(snippet) - end_line = end_line - total_trimmed_lines + {trimmed, [last_line | _] = reversed_lines} = + snippet + |> String.split("\n") + |> Enum.reverse() + |> Enum.split_while(&(&1 == "")) - end_column = - lines - |> Enum.fetch!(end_line - line_offset - 1) - |> String.length() - |> Kernel.+(1) + end_line = end_line - length(trimmed) + end_column = String.length(last_line) + 1 start_pos = {line, column} end_pos = {end_line, end_column} @@ -1344,10 +1314,9 @@ defmodule TokenMissingError do Exception.format_snippet( start_pos, end_pos, - line_offset, description, file, - lines, + Enum.reverse(reversed_lines), start_message, end_message ) @@ -1369,13 +1338,6 @@ defmodule TokenMissingError do format_message(file, line, column, snippet) end - defp handle_trailing_newlines(snippet) do - trimmed_snippet = String.trim_trailing(snippet, "\n") - total_trimmed_newlines = String.length(snippet) - String.length(trimmed_snippet) - lines = String.split(trimmed_snippet, "\n") - {lines, total_trimmed_newlines} - end - defp format_message(file, line, column, message) do location = Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) "token missing on " <> location <> "\n" <> message diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 96d9ea6ec5b..96da99388e6 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -26,7 +26,7 @@ print_warning(Position, File, Message) -> %% Called by Module.ParallelChecker. print_warning(Message, Diagnostic) -> #{file := File, position := Position, stacktrace := S} = Diagnostic, - Snippet = get_snippet(File, Position), + Snippet = read_snippet(File, Position), Span = get_span(Diagnostic), Output = format_snippet(Position, File, Message, Snippet, warning, S, Span), io:put_chars(standard_error, [Output, $\n, $\n]). @@ -34,7 +34,7 @@ print_warning(Message, Diagnostic) -> %% Called by Module.ParallelChecker. print_warning_group(Message, [Diagnostic | Others]) -> #{file := File, position := Position, stacktrace := S} = Diagnostic, - Snippet = get_snippet(File, Position), + Snippet = read_snippet(File, Position), Span = get_span(Diagnostic), Formatted = format_snippet(Position, File, Message, Snippet, warning, S, Span), LineNumber = extract_line(Position), @@ -49,11 +49,11 @@ print_warning_group(Message, [Diagnostic | Others]) -> get_span(#{span := nil}) -> nil; get_span(#{span := Span}) -> Span. -get_snippet(nil, _Position) -> +read_snippet(nil, _Position) -> nil; -get_snippet(<<"nofile">>, _Position) -> +read_snippet(<<"nofile">>, _Position) -> nil; -get_snippet(File, Position) -> +read_snippet(File, Position) -> LineNumber = extract_line(Position), get_file_line(File, LineNumber). @@ -80,7 +80,7 @@ traverse_file_line(IoDevice, N) -> print_diagnostic(#{severity := Severity, message := M, stacktrace := Stacktrace, position := P, file := F} = Diagnostic, ReadSnippet) -> Snippet = case ReadSnippet of - true -> get_snippet(F, P); + true -> read_snippet(F, P); false -> nil end, @@ -320,10 +320,8 @@ parse_error(Location, File, Error, <<>>, Input) -> <<"syntax error before: ">> -> <<"syntax error: expression is incomplete">>; _ -> <> end, - case lists:keytake(error_type, 1, Location) of - {value, {error_type, unclosed_delimiter}, Loc} -> raise_unclosed_delimiter(Loc, File, Input, Message); - _ -> raise_snippet(Location, File, Input, 'Elixir.TokenMissingError', Message) - end; + + raise_snippet(Location, File, Input, 'Elixir.TokenMissingError', Message); %% Show a nicer message for end of line parse_error(Location, File, <<"syntax error before: ">>, <<"eol">>, Input) -> @@ -398,8 +396,10 @@ parse_error(Location, File, <<"syntax error before: ">>, <<$$, Char/binary>>, In parse_error(Location, File, Error, Token, Input) when is_binary(Error), is_binary(Token) -> Message = <>, case lists:keytake(error_type, 1, Location) of - {value, {error_type, mismatched_delimiter}, Loc} -> raise_mismatched_delimiter(Loc, File, Input, Message); - _ -> raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message) + {value, {error_type, mismatched_delimiter}, Loc} -> + raise_snippet(Loc, File, Input, 'Elixir.MismatchedDelimiterError', Message); + _ -> + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message) end. parse_erl_term(Term) -> @@ -407,18 +407,6 @@ parse_erl_term(Term) -> {ok, Parsed} = erl_parse:parse_term(Tokens ++ [{dot, 1}]), Parsed. -raise_mismatched_delimiter(Location, File, Input, Message) -> - {InputString, StartLine, StartColumn} = Input, - Snippet = indent(elixir_utils:characters_to_binary(InputString), StartColumn), - Opts = [{file, File}, {line_offset, StartLine - 1}, {snippet, Snippet} | Location], - raise('Elixir.MismatchedDelimiterError', Message, Opts). - -raise_unclosed_delimiter(Location, File, Input, Message) -> - {InputString, StartLine, StartColumn} = Input, - Snippet = indent(elixir_utils:characters_to_binary(InputString), StartColumn), - Opts = [{line_offset, StartLine - 1}, {file, File}, {snippet, Snippet} | Location], - raise('Elixir.TokenMissingError', Message, Opts). - raise_reserved(Location, File, Input, Keyword) -> raise_snippet(Location, File, Input, 'Elixir.SyntaxError', <<"syntax error before: ", Keyword/binary, ". \"", Keyword/binary, "\" is a " @@ -426,32 +414,43 @@ raise_reserved(Location, File, Input, Keyword) -> "it can't be used as a variable or be defined nor invoked as a regular function">>). raise_snippet(Location, File, Input, Kind, Message) when is_binary(File) -> - {InputString, StartLine, StartColumn} = Input, - Snippet = snippet_line(InputString, Location, StartLine), - raise(Kind, Message, [{file, File}, {snippet, indent(Snippet, StartColumn)} | Location]). - -indent(nil, _StartColumn) -> nil; -indent(Snippet, StartColumn) when StartColumn > 1 -> - Prefix = binary:copy(<<" ">>, StartColumn - 1), - Replaced = binary:replace(Snippet, <<"\n">>, <>, [global]), - <>; -indent(Snippet, _StartColumn) -> Snippet. - -snippet_line(InputString, Location, StartLine) -> - {line, Line} = lists:keyfind(line, 1, Location), + Snippet = cut_snippet(Location, Input), + raise(Kind, Message, [{file, File}, {snippet, Snippet} | Location]). + +cut_snippet(Location, Input) -> case lists:keyfind(column, 1, Location) of {column, _} -> - Lines = string:split(InputString, "\n", all), - Snippet = (lists:nth(Line - StartLine + 1, Lines)), - case string:trim(Snippet, leading) of - [] -> nil; - _ -> elixir_utils:characters_to_binary(Snippet) + {line, Line} = lists:keyfind(line, 1, Location), + + case lists:keyfind(end_line, 1, Location) of + {end_line, EndLine} -> + cut_snippet(Input, Line, EndLine - Line + 1); + + false -> + Snippet = cut_snippet(Input, Line, 1), + case string:trim(Snippet, leading) of + <<>> -> nil; + _ -> Snippet + end end; false -> nil end. +cut_snippet({InputString, StartLine, StartColumn}, Line, Span) -> + %% In case the code is indented, we need to add the indentation back + %% for the snippets to match the reported columns. + Indent = binary:copy(<<" ">>, StartColumn - 1), + Lines = string:split(InputString, "\n", all), + [Head | Tail] = lists:nthtail(Line - StartLine, Lines), + IndentedTail = indent_n(Tail, Span - 1, <<"\n", Indent/binary>>), + elixir_utils:characters_to_binary([Indent, Head, IndentedTail]). + +indent_n([], _Count, _Indent) -> []; +indent_n(_Lines, 0, _Indent) -> []; +indent_n([H | T], Count, Indent) -> [Indent, H | indent_n(T, Count - 1, Indent)]. + %% Helpers prefix(warning) -> highlight(<<"warning:">>, warning); diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 03697a1953e..0fb0cab5951 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -148,7 +148,6 @@ tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLi Message = "missing terminator: ~ts", Formatted = io_lib:format(Message, [End]), Meta = [ - {error_type, unclosed_delimiter}, {opening_delimiter, Start}, {expected_delimiter, End}, {line, StartLine}, @@ -1350,7 +1349,6 @@ interpolation_error(Reason, Rest, Scope, Tokens, Extension, Args, Line, Column, interpolation_format({string, EndLine, EndColumn, Message, Token}, Extension, Args, Line, Column, Opening, Closing) -> Meta = [ - {error_type, unclosed_delimiter}, {opening_delimiter, list_to_atom(Opening)}, {expected_delimiter, list_to_atom(Closing)}, {line, Line}, From 1b7c73273df0282ec7052ff5711f1b60400415d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 22 Dec 2023 14:31:44 +0100 Subject: [PATCH 107/185] Last pass over anti-patterns --- .../pages/anti-patterns/code-anti-patterns.md | 91 ++++++++----------- .../anti-patterns/design-anti-patterns.md | 65 +------------ .../anti-patterns/macro-anti-patterns.md | 2 +- .../anti-patterns/process-anti-patterns.md | 29 +++--- 4 files changed, 61 insertions(+), 126 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index 2d0147f5a6a..e93f314057a 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -309,9 +309,13 @@ There are few known exceptions to this anti-pattern: #### Problem -In Elixir, it is possible to access values from `Map`s, which are key-value data structures, either statically or dynamically. When a key is expected to exist in the map, it must be accessed using the `map.key` notation, which asserts the key exists. If the key does not exist, an exception is raised (and in some situations also compiler warnings), allowing developers to catch bugs early on. +In Elixir, it is possible to access values from `Map`s, which are key-value data structures, either statically or dynamically. -`map[:key]` must be used with optional keys. This way, if the informed key does not exist, `nil` is returned. When used with required keys, this return can be confusing and allow `nil` values to pass through the system, while `map.key` would raise upfront. In this way, this anti-pattern may cause bugs in the code. +When a key is expected to exist in a map, it must be accessed using the `map.key` notation, making it clear to developers (and the compiler) that the key must exist. If the key does not exist, an exception is raised (and in some cases also compiler warnings). This is also known as the static notation, as the key is known at the time of writing the code. + +When a key is optional, the `map[:key]` notation must be used instead. This way, if the informed key does not exist, `nil` is returned. This is the dynamic notation, as it also supports dynamic key access, such as `map[some_var]`. + +When you use `map[:key]` to access a key that always exists in the map, you are making the code less clear for developers and for the compiler, as they now need to work with the assumption the key may not be there. This mismatch may also make it harder to track certain bugs. If the key is unexpected missing, you will have a `nil` value propagate through the system, instead of raising on map access. #### Example @@ -321,8 +325,6 @@ The function `plot/1` tries to draw a graphic to represent the position of a poi defmodule Graphics do def plot(point) do # Some other code... - - # Dynamic access to use point values {point[:x], point[:y], point[:z]} end end @@ -331,92 +333,77 @@ end ```elixir iex> point_2d = %{x: 2, y: 3} %{x: 2, y: 3} -iex> point_3d = %{x: 5, y: 6, z: nil} -%{x: 5, y: 6, z: nil} +iex> point_3d = %{x: 5, y: 6, z: 7} +%{x: 5, y: 6, z: 7} iex> Graphics.plot(point_2d) -{2, 3, nil} # <= ambiguous return +{2, 3, nil} iex> Graphics.plot(point_3d) -{5, 6, nil} +{5, 6, 7} +``` + +Given we want to plot both 2D and 3D points, the behaviour above is expected. But what happens if we forget to pass a point with either `:x` or `:y`? + +```elixir +iex> bad_point = %{y: 3, z: 4} +%{y: 3, z: 4} +iex> Graphics.plot(bad_point) +{nil, 3, 4} ``` -As can be seen in the example above, even when the key `:z` does not exist in the map (`point_2d`), dynamic access returns the value `nil`. This return can be dangerous because of its ambiguity. It is not possible to conclude from it whether the map has the key `:z` or not. If the function relies on the return value to make decisions about how to plot a point, this can be problematic and even cause errors when testing the code. +The behaviour above is unexpected because our function should not work with points without a `:x` key. This leads to subtle bugs, as we may now pass `nil` to another function, instead of raising early on. #### Refactoring -To remove this anti-pattern, whenever accessing an existing key of `Atom` type in the map, replace the dynamic `map[:key]` syntax by the static `map.key` notation. This way, when a non-existent key is accessed, Elixir raises an error immediately, allowing developers to find bugs faster. The next code illustrates the refactoring of `plot/1`, removing this anti-pattern: +To remove this anti-pattern, we must use the dynamic `map[:key]` syntax and the static `map.key` notation according to our requirements. We expect `:x` and `:y` to always exist, but not `:z`. The next code illustrates the refactoring of `plot/1`, removing this anti-pattern: ```elixir defmodule Graphics do def plot(point) do # Some other code... - - # Strict access to use point values - {point.x, point.y, point.z} + {point.x, point.y, point[:z]} end end ``` ```elixir -iex> point_2d = %{x: 2, y: 3} -%{x: 2, y: 3} -iex> point_3d = %{x: 5, y: 6, z: nil} -%{x: 5, y: 6, z: nil} iex> Graphics.plot(point_2d) -** (KeyError) key :z not found in: %{x: 2, y: 3} # <= explicitly warns that - graphic.ex:6: Graphics.plot/1 # <= the :z key does not exist! -iex> Graphics.plot(point_3d) -{5, 6, nil} +{2, 3, nil} +iex> Graphics.plot(bad_point) +** (KeyError) key :x not found in: %{y: 3, z: 4} # <= explicitly warns that + graphic.ex:4: Graphics.plot/1 # <= the :z key does not exist! ``` Overall, the usage of `map.key` and `map[:key]` encode important information about your data structure, allowing developers to be clear about their intent. See both `Map` and `Access` module documentation for more information and examples. -An even simpler alternative to refactor this anti-pattern is to use pattern matching: +An alternative to refactor this anti-pattern is to use pattern matching, defining explicit clauses for 2d vs 3d points: ```elixir defmodule Graphics do - def plot(%{x: x, y: y, z: z}) do + # 2d + def plot(%{x: x, y: y}) do # Some other code... + {x, y} + end - # Strict access to use point values + # 3d + def plot(%{x: x, y: y, z: z}) do + # Some other code... {x, y, z} end end ``` -```elixir -iex> point_2d = %{x: 2, y: 3} -%{x: 2, y: 3} -iex> point_3d = %{x: 5, y: 6, z: nil} -%{x: 5, y: 6, z: nil} -iex> Graphics.plot(point_2d) -** (FunctionClauseError) no function clause matching in Graphics.plot/1 - graphic.ex:2: Graphics.plot/1 # <= the :z key does not exist! -iex> Graphics.plot(point_3d) -{5, 6, nil} -``` - -Pattern-matching is specially useful when matching over multiple keys at once and also when you want to match and assert on the values of a map. +Pattern-matching is specially useful when matching over multiple keys as well as on the values themselves at once. -Another alternative is to use structs. By default, structs only support static access to its fields: +Another option is to use structs. By default, structs only support static access to its fields. In such scenarios, you may consider defining structs for both 2D and 3D points: ```elixir -defmodule Point.2D do +defmodule Point2D do @enforce_keys [:x, :y] defstruct [x: nil, y: nil] end ``` -```elixir -iex> point = %Point.2D{x: 2, y: 3} -%Point.2D{x: 2, y: 3} -iex> point.x # <= strict access to use point values -2 -iex> point.z # <= trying to access a non-existent key -** (KeyError) key :z not found in: %Point{x: 2, y: 3} -iex> point[:x] # <= by default, struct does not support dynamic access -** (UndefinedFunctionError) ... (Point does not implement the Access behaviour) -``` - Generally speaking, structs are useful when sharing data structures across modules, at the cost of adding a compile time dependency between these modules. If module `A` uses a struct defined in module `B`, `A` must be recompiled if the fields in the struct `B` change. #### Additional remarks @@ -498,7 +485,7 @@ case some_function(arg) do end ``` -In particular, avoid matching solely on `_`, as shown below, as it is less clear in intent and it may hide bugs if `some_function/1` adds new return values in the future: +In particular, avoid matching solely on `_`, as shown below: ```elixir case some_function(arg) do @@ -507,6 +494,8 @@ case some_function(arg) do end ``` + Matching on `_` is less clear in intent and it may hide bugs if `some_function/1` adds new return values in the future. + #### Additional remarks This anti-pattern was formerly known as [Speculative assumptions](https://github.com/lucasvegi/Elixir-Code-Smells#speculative-assumptions). diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index aedc3a7a53d..98854e1511f 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -1,7 +1,6 @@ # Design-related anti-patterns -This document outlines potential anti-patterns related to your modules, functions, and the role they -play within a codebase. +This document outlines potential anti-patterns related to your modules, functions, and the role they play within a codebase. ## Alternative return types @@ -242,66 +241,6 @@ defmodule MyApp do end ``` -## Propagating invalid data - -#### Problem - -This anti-pattern refers to a function that does not validate its parameters and propagates them to other functions, which can produce internal unexpected behavior. When an error is raised inside a function due to an invalid parameter value, it can be confusing for developers and make it harder to locate and fix the error. - -#### Example - -An example of this anti-pattern is when a function receives an invalid parameter and then passes it to other functions, either in the same library or in a third-party library. This can cause an error to be raised deep inside the call stack, which may be confusing for the developer who is working with invalid data. As shown next, the function `foo/1` is a user-facing API which doesn't validate its parameters at the boundary. In this way, it is possible that invalid data will be passed through, causing an error that is obscure and hard to debug. - -```elixir -defmodule MyLibrary do - def foo(invalid_data) do - # Some other code... - - MyLibrary.Internal.sum(1, invalid_data) - end -end -``` - -```elixir -iex> MyLibrary.foo(2) -3 -iex> MyLibrary.foo("José") # With invalid data -** (ArithmeticError) bad argument in arithmetic expression: 1 + "José" - :erlang.+(1, "José") - my_library.ex:4: MyLibrary.Internal.sum/2 -``` - -#### Refactoring - -To remove this anti-pattern, the client code must validate input parameters at the boundary with the user, via guard clauses, pattern matching, or conditionals. This prevents errors from occurring elsewhere in the call stack, making them easier to understand and debug. This refactoring also allows libraries to be implemented without worrying about creating internal protection mechanisms. The next code snippet illustrates the refactoring of `foo/1`, removing this anti-pattern: - -```elixir -defmodule MyLibrary do - def foo(data) when is_integer(data) do - # Some other code - - MyLibrary.Internal.sum(1, data) - end -end -``` - -```elixir -iex> MyLibrary.foo(2) # With valid data -3 -iex> MyLibrary.foo("José") # With invalid data -** (FunctionClauseError) no function clause matching in MyLibrary.foo/1. -The following arguments were given to MyLibrary.foo/1: - - # 1 - "José" - - my_library.ex:2: MyLibrary.foo/1 -``` - -#### Additional remarks - -This anti-pattern was formerly known as [Working with invalid data](https://github.com/lucasvegi/Elixir-Code-Smells#working-with-invalid-data). - ## Unrelated multi-clause function #### Problem @@ -377,7 +316,7 @@ end You can see this pattern in practice within Elixir itself. The `+/2` operator can add `Integer`s and `Float`s together, but not `String`s, which instead use the `<>/2` operator. In this sense, it is reasonable to handle integers and floats in the same operation, but strings are unrelated enough to deserve their own function. -You will also find examples in Elixir of functions that work with any struct, such as `struct/2`: +You will also find examples in Elixir of functions that work with any struct, which would seemingly be an occurrence of this anti-pattern, such as `struct/2`: ```elixir iex> struct(URI.parse("/foo/bar"), path: "/bar/baz") diff --git a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md index 26a0c603027..1885965318c 100644 --- a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md @@ -63,7 +63,7 @@ end #### Problem -**Macros** are powerful meta-programming mechanisms that can be used in Elixir to extend the language. While using macros is not an anti-pattern in itself, this meta-programming mechanism should only be used when absolutely necessary. Whenever a macro is used, but it would have been possible to solve the same problem using functions or other existing Elixir structures, the code becomes unnecessarily more complex and less readable. Because macros are more difficult to implement and reason about, their indiscriminate use can compromise the evolution of a system, reducing its maintainability. +*Macros* are powerful meta-programming mechanisms that can be used in Elixir to extend the language. While using macros is not an anti-pattern in itself, this meta-programming mechanism should only be used when absolutely necessary. Whenever a macro is used, but it would have been possible to solve the same problem using functions or other existing Elixir structures, the code becomes unnecessarily more complex and less readable. Because macros are more difficult to implement and reason about, their indiscriminate use can compromise the evolution of a system, reducing its maintainability. #### Example diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md index c27c2ea0555..6c513b81678 100644 --- a/lib/elixir/pages/anti-patterns/process-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -6,7 +6,7 @@ This document outlines potential anti-patterns related to processes and process- #### Problem -This anti-pattern refers to code that is unnecessarily organized by processes. A process itself does not represent an anti-pattern, but it should only be used to model runtime properties (such as concurrency, access to shared resources, and event scheduling). When you use a process for code organization, it can create bottlenecks in the system. +This anti-pattern refers to code that is unnecessarily organized by processes. A process itself does not represent an anti-pattern, but it should only be used to model runtime properties (such as concurrency, access to shared resources, error isolation, etc). When you use a process for code organization, it can create bottlenecks in the system. #### Example @@ -261,7 +261,9 @@ defmodule Counter do use GenServer @doc "Starts a counter process." - def start(initial_value, name \\ __MODULE__) when is_integer(initial_value) do + def start_link(opts \\ []) do + initial_valye = Keyword.get(opts, :initial_value, 0) + name = Keywoird.get(opts, :name, __MODULE__) GenServer.start(__MODULE__, initial_value, name: name) end @@ -271,7 +273,7 @@ defmodule Counter do end @doc "Bumps the value of the given counter." - def bump(value, pid_name \\ __MODULE__) do + def bump(pid_name \\ __MODULE__, value) do GenServer.call(pid_name, {:bump, value}) end @@ -292,17 +294,17 @@ end ``` ```elixir -iex> Counter.start(0) +iex> Counter.start_link() {:ok, #PID<0.115.0>} iex> Counter.get() 0 -iex> Counter.start(15, :other_counter) +iex> Counter.start_link(initial_value: 15, name: :other_counter) {:ok, #PID<0.120.0>} iex> Counter.get(:other_counter) 15 -iex> Counter.bump(-3, :other_counter) +iex> Counter.bump(:other_counter, -3) 12 -iex> Counter.bump(7) +iex> Counter.bump(Counter, 7) 7 ``` @@ -317,8 +319,13 @@ defmodule SupervisedProcess.Application do @impl true def start(_type, _args) do children = [ - %{id: Counter, start: {Counter, :start, [0]}}, - %{id: :other_counter, start: {Counter, :start, [0, :other_counter]}} + # With the default values for counter and name + Counter, + # With custom values for counter, name, and a custom ID + Supervisor.child_spec( + {Counter, name: :other_counter, initial_value: 15}, + id: :other_counter + ) ] Supervisor.start_link(children, strategy: :one_for_one, name: App.Supervisor) @@ -332,8 +339,8 @@ iex> Supervisor.count_children(App.Supervisor) iex> Counter.get(Counter) 0 iex> Counter.get(:other_counter) -0 -iex> Counter.bump(7, Counter) +15 +iex> Counter.bump(Counter, 7) 7 iex> Supervisor.terminate_child(App.Supervisor, Counter) iex> Supervisor.count_children(App.Supervisor) # Only one active child From 38ddc98fef7faf6f4666703dcd9c99105e6388d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 22 Dec 2023 17:45:12 +0100 Subject: [PATCH 108/185] Release v1.16.0 --- CHANGELOG.md | 73 +++++++++++++++++++------------------------------- VERSION | 2 +- bin/elixir | 2 +- bin/elixir.bat | 2 +- 4 files changed, 30 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 939c33df786..19ff0980edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,7 @@ A huge thank you to Vinícius Müller for working on the new diagnostics. ## Revamped documentation -Elixir's Getting Started guided has been made part of the Elixir repository and incorporated into ExDoc. This was an opportunity to revisit and unify all official guides and references. +Elixir's Getting Started guides have been made part of the Elixir repository and incorporated into ExDoc. This was an opportunity to revisit and unify all official guides and references. We have also incorporated and extended the work on [Understanding Code Smells in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf), by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/), into the official document in the form of anti-patterns. The anti-patterns are divided into four categories: code-related, design-related, process-related, and meta-programming. Our goal is to give all developers examples of potential anti-patterns, with context and examples on how to improve their codebases. @@ -78,51 +78,7 @@ Another [ExDoc](https://github.com/elixir-lang/ex_doc) feature we have incorpora Finally, we have started enriching our documentation with [Mermaid.js](https://mermaid.js.org/) diagrams. You can find examples in the [GenServer](https://hexdocs.pm/elixir/main/GenServer.html) and [Supervisor](https://hexdocs.pm/elixir/main/Supervisor.html) docs. -## v1.16.0-rc.1 (2023-12-12) - -### 1. Enhancements - -#### Elixir - - * [Code] Add `:emit_warnings` for `Code.string_to_quoted/2` - * [File] Add `:offset` option to `File.stream!/2` - * [Kernel] Auto infer size of matched variable in bitstrings - * [Kernel] Preserve column information when translating typespecs - * [String] Add `String.replace_invalid/2` - -#### Logger - - * [Logger] Add `Logger.levels/0` - -#### Mix - - * [mix archive.install] Support `--sparse` option - * [mix compile.app] Warn if both `:applications` and `:extra_applications` are used - * [mix compile.elixir] Pass original exception down to diagnostic `:details` when possible - * [mix deps.clean] Emit a warning instead of crashing when a dependency cannot be removed - * [mix escript.install] Support `--sparse` option - * [mix release] Include `include/` directory in releases - -### 2. Bug fixes - -#### Elixir - - * [Code] Keep quotes for atom keys in formatter - * [Kernel] Re-enabled compiler optimizations for top level functions in scripts (disabled in v1.14.0 but shouldn't impact most programs) - * [Macro] Address exception on `Macro.to_string/1` for certain ASTs - * [Module] Make sure file and position information is included in several module warnings (regression) - * [Path] Lazily evaluate `File.cwd!/0` in `Path.expand/1` and `Path.absname/1` - -#### IEx - - * [IEx.Pry] Fix prying functions with only literals in their body - -#### Mix - - * [mix archive.install] Restore code paths after archive.install - * [mix escript.install] Restore code paths after escript.install - -## v1.16.0-rc.0 (2023-10-31) +## v1.16.0 (2023-12-22) ### 1. Enhancements @@ -132,10 +88,14 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m #### Elixir + * [Code] Add `:emit_warnings` for `Code.string_to_quoted/2` * [Code] Automatically include columns in parsing options * [Code] Introduce `MismatchedDelimiterError` for handling mismatched delimiter exceptions * [Code.Fragment] Handle anonymous calls in fragments * [Code.Formatter] Trim trailing whitespace on heredocs with `\r\n` + * [File] Add `:offset` option to `File.stream!/2` + * [Kernel] Auto infer size of matched variable in bitstrings + * [Kernel] Preserve column information when translating typespecs * [Kernel] Suggest module names based on suffix and casing errors when the module does not exist in `UndefinedFunctionError` * [Kernel.ParallelCompiler] Introduce `Kernel.ParallelCompiler.pmap/2` to compile multiple additional entries in parallel * [Kernel.SpecialForms] Warn if `True`/`False`/`Nil` are used as aliases and there is no such alias @@ -143,35 +103,56 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m * [Module] Add support for `@nifs` annotation from Erlang/OTP 25 * [Module] Add support for missing `@dialyzer` configuration * [String] Update to Unicode 15.1.0 + * [String] Add `String.replace_invalid/2` * [Task] Add `:limit` option to `Task.yield_many/2` +#### Logger + + * [Logger] Add `Logger.levels/0` + #### Mix * [mix] Add `MIX_PROFILE` to profile a list of comma separated tasks + * [mix archive.install] Support `--sparse` option + * [mix compile.app] Warn if both `:applications` and `:extra_applications` are used + * [mix compile.elixir] Pass original exception down to diagnostic `:details` when possible * [mix compile.elixir] Optimize scenario where there are thousands of files in `lib/` and one of them is changed + * [mix deps.clean] Emit a warning instead of crashing when a dependency cannot be removed + * [mix escript.install] Support `--sparse` option + * [mix release] Include `include/` directory in releases * [mix test] Allow testing multiple file:line at once, such as `mix test test/foo_test.exs:13 test/bar_test.exs:27` ### 2. Bug fixes #### Elixir + * [Code] Keep quotes for atom keys in formatter * [Code.Fragment] Fix crash in `Code.Fragment.surround_context/2` when matching on `->` * [IO] Raise when using `IO.binwrite/2` on terminated device (mirroring `IO.write/2`) * [Kernel] Do not expand aliases recursively (the alias stored in Macro.Env is already expanded) * [Kernel] Ensure `dbg` module is a compile-time dependency * [Kernel] Warn when a private function or macro uses `unquote/1` and the function/macro itself is unused + * [Kernel] Re-enabled compiler optimizations for top level functions in scripts (disabled in v1.14.0 but shouldn't impact most programs) * [Kernel] Do not define an alias for nested modules starting with `Elixir.` in their definition * [Kernel.ParallelCompiler] Consider a module has been defined in `@after_compile` callbacks to avoid deadlocks + * [Macro] Address exception on `Macro.to_string/1` for certain ASTs + * [Path] Lazily evaluate `File.cwd!/0` in `Path.expand/1` and `Path.absname/1` * [Path] Ensure `Path.relative_to/2` returns a relative path when the given argument does not share a common prefix with `cwd` #### ExUnit * [ExUnit] Raise on incorrectly dedented doctests +#### IEx + + * [IEx.Pry] Fix prying functions with only literals in their body + #### Mix + * [mix archive.install] Restore code paths after `mix archive.install` * [mix compile] Ensure files with duplicate modules are recompiled whenever any of the files change * [mix compile] Update Mix compiler diagnostics documentation and typespecs to match the Elixir compiler behaviour where both lines and columns start from one (before it inaccurately said that columns started from zero) + * [mix escript.install] Restore code paths after `mix escript.install` ### 3. Soft deprecations (no warnings emitted) diff --git a/VERSION b/VERSION index f45d77ad4ce..71bd5d9ee64 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.0-rc.1 \ No newline at end of file +1.16.0 \ No newline at end of file diff --git a/bin/elixir b/bin/elixir index 01259f98449..7d660e025e9 100755 --- a/bin/elixir +++ b/bin/elixir @@ -1,7 +1,7 @@ #!/bin/sh set -e -ELIXIR_VERSION=1.16.0-rc.1 +ELIXIR_VERSION=1.16.0 if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then cat <&2 diff --git a/bin/elixir.bat b/bin/elixir.bat index f29bfe0a190..604dec081b0 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -1,6 +1,6 @@ @if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) -set ELIXIR_VERSION=1.16.0-rc.1 +set ELIXIR_VERSION=1.16.0 setlocal enabledelayedexpansion if ""%1""=="""" if ""%2""=="""" goto documentation From 91375778cbd014d5cea05edf8529e400c5251c4c Mon Sep 17 00:00:00 2001 From: Alex Martsinovich Date: Sat, 23 Dec 2023 01:41:00 +0000 Subject: [PATCH 109/185] Fix example in non-assertive map access antipattern (#13201) --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index e93f314057a..a7706cf8b41 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -379,17 +379,17 @@ An alternative to refactor this anti-pattern is to use pattern matching, definin ```elixir defmodule Graphics do - # 2d - def plot(%{x: x, y: y}) do - # Some other code... - {x, y} - end - # 3d def plot(%{x: x, y: y, z: z}) do # Some other code... {x, y, z} end + + # 2d + def plot(%{x: x, y: y}) do + # Some other code... + {x, y} + end end ``` From f84bc19bf294981d7f159663a1c430cd155c67f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 24 Dec 2023 14:54:38 +0100 Subject: [PATCH 110/185] Additional clarity on long list of parameters anti-patterns --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index a7706cf8b41..340256af1c4 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -251,7 +251,9 @@ end #### Refactoring -To address this anti-pattern, related arguments can be grouped using maps, structs, or even tuples. This effectively reduces the number of arguments, simplifying the function's interface. In the case of `loan/6`, its arguments were grouped into two different maps, thereby reducing its arity to `loan/2`: +To address this anti-pattern, related arguments can be grouped using key-value data structures, such as maps, structs, or even keyword lists in the case of optional arguments. This effectively reduces the number of arguments and the key-value data structures adds clarity to the caller. + +For this particular example, the arguments to `loan/6` can be grouped into two different maps, thereby reducing its arity to `loan/2`: ```elixir defmodule Library do From 97c608c3460c46d533b3f1df80c6efdbc0935448 Mon Sep 17 00:00:00 2001 From: Artem Solomatin Date: Mon, 25 Dec 2023 22:56:00 +0300 Subject: [PATCH 111/185] Fix typo in design-anti-patterns doc (#13207) --- lib/elixir/pages/anti-patterns/design-anti-patterns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index 98854e1511f..a143b62e3b1 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -418,7 +418,7 @@ def project do end ``` -Additonally, if a Mix task is available, you can also accept these options as command line arguments (see `OptionParser`): +Additionally, if a Mix task is available, you can also accept these options as command line arguments (see `OptionParser`): ```bash mix linter --output-file /path/to/output.json --verbosity 3 From 298acd1cdbee643cfc1a29d0a2ecc74028ac06a9 Mon Sep 17 00:00:00 2001 From: Travis Vander Hoop Date: Tue, 26 Dec 2023 02:55:05 -0600 Subject: [PATCH 112/185] Fix typos and tweak language in anti-pattern docs (#13208) --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 2 +- lib/elixir/pages/anti-patterns/design-anti-patterns.md | 6 +++--- lib/elixir/pages/anti-patterns/macro-anti-patterns.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index 340256af1c4..d97a2534b9e 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -317,7 +317,7 @@ When a key is expected to exist in a map, it must be accessed using the `map.key When a key is optional, the `map[:key]` notation must be used instead. This way, if the informed key does not exist, `nil` is returned. This is the dynamic notation, as it also supports dynamic key access, such as `map[some_var]`. -When you use `map[:key]` to access a key that always exists in the map, you are making the code less clear for developers and for the compiler, as they now need to work with the assumption the key may not be there. This mismatch may also make it harder to track certain bugs. If the key is unexpected missing, you will have a `nil` value propagate through the system, instead of raising on map access. +When you use `map[:key]` to access a key that always exists in the map, you are making the code less clear for developers and for the compiler, as they now need to work with the assumption the key may not be there. This mismatch may also make it harder to track certain bugs. If the key is unexpectedly missing, you will have a `nil` value propagate through the system, instead of raising on map access. #### Example diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index a143b62e3b1..e74eedd5d3c 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -245,11 +245,11 @@ end #### Problem -Using multi-clause functions in Elixir, to group functions of the same name, is a powerful Elixir feature. However, some developers may abuse this feature to group *unrelated* functionality, which configures an anti-pattern. +Using multi-clause functions is a powerful Elixir feature. However, some developers may abuse this feature to group *unrelated* functionality, which is an anti-pattern. #### Example -A frequent example of this usage of multi-clause functions is when developers mix unrelated business logic into the same function definition, in a way the behaviour of each clause is completely distinct from the other ones. Such functions often have too broad specifications, making it difficult for other developers to understand and maintain them. +A frequent example of this usage of multi-clause functions occurs when developers mix unrelated business logic into the same function definition, in a way that the behaviour of each clause becomes completely distinct from the others. Such functions often have too broad specifications, making it difficult for other developers to understand and maintain them. Some developers may use documentation mechanisms such as `@doc` annotations to compensate for poor code readability, however the documentation itself may end-up full of conditionals to describe how the function behaves for each different argument combination. This is a good indicator that the clauses are ultimately unrelated. @@ -274,7 +274,7 @@ If updating an animal is completely different from updating a product and requir #### Refactoring -As shown below, a possible solution to this anti-pattern is to break the business rules that are mixed up in a single unrelated multi-clause function in simple functions. Each function can have a specific name and `@doc`, describing its behavior and parameters received. While this refactoring sounds simple, it can impact the function's current users, so be careful! +As shown below, a possible solution to this anti-pattern is to break the business rules that are mixed up in a single unrelated multi-clause function in simple functions. Each function can have a specific name and `@doc`, describing its behavior and parameters received. While this refactoring sounds simple, it can impact the function's callers, so be careful! ```elixir @doc """ diff --git a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md index 1885965318c..b8be9d93672 100644 --- a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md @@ -165,7 +165,7 @@ error: imported ModuleA.foo/0 conflicts with local function #### Refactoring -To remove this anti-pattern, we recommend library authors to avoid providing `__using__/1` callbacks whenever it can be replaced by `alias/1` or `import/1` directives. In the following code, we assume `use Library` is no longer available and `ClientApp` was refactored in this way, and with that, the code is clearer and the conflict as previously shown no longer exists: +To remove this anti-pattern, we recommend library authors avoid providing `__using__/1` callbacks whenever it can be replaced by `alias/1` or `import/1` directives. In the following code, we assume `use Library` is no longer available and `ClientApp` was refactored in this way, and with that, the code is clearer and the conflict as previously shown no longer exists: ```elixir defmodule ClientApp do From 194661197eea9711abe48711e7184aa5a4aeb556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 26 Dec 2023 09:55:51 +0100 Subject: [PATCH 113/185] Add more examples to app config anti-pattern (#13204) --- .../anti-patterns/design-anti-patterns.md | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index e74eedd5d3c..fd1aa23a987 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -374,7 +374,7 @@ iex> DashSplitter.split("Lucas-Francisco-da-Matta-Vegi") #### Refactoring -To remove this anti-pattern and make the library more adaptable and flexible, this type of configuration must be performed via parameters in function calls. The code shown below performs the refactoring of the `split/1` function by accepting [keyword lists](`Keyword`) as a new optional parameter. With this new parameter, it is possible to modify the default behavior of the function at the time of its call, allowing multiple different ways of using `split/2` within the same application: +To remove this anti-pattern, this type of configuration should be performed using a parameter passed to the function. The code shown below performs the refactoring of the `split/1` function by accepting [keyword lists](`Keyword`) as a new optional parameter. With this new parameter, it is possible to modify the default behavior of the function at the time of its call, allowing multiple different ways of using `split/2` within the same application: ```elixir defmodule DashSplitter do @@ -392,7 +392,55 @@ iex> DashSplitter.split("Lucas-Francisco-da-Matta-Vegi") #<= default config is u ["Lucas", "Francisco-da-Matta-Vegi"] ``` -#### Additional Remarks +Of course, not all uses of the application environment by libraries are incorrect. One example is using configuration to replace a component (or dependency) of a library by another that must behave the exact same. Consider a library that needs to parse CSV files. The library author may pick one package to use as default parser but allow its users to swap to different implementations via the application environment. At the end of the day, choosing a different CSV parser should not change the outcome, and library authors can even enforce this by [defining behaviours](../references/typespecs.md#behaviours) with the exact semantics they expect. + +#### Additional remarks: Supervision trees + +In practice, libraries may require additional configuration beyond keyword lists. For example, if a library needs to start a supervision tree, how can the user of said library customize its supervision tree? Given the supervision tree itself is global (as it belongs to the library), library authors may be tempted to use the application configuration once more. + +One solution is for the library to provide its own child specification, instead of starting the supervision tree itself. This allows the user to start all necessary processes under its own supervision tree, potentially passing custom configuration options during initialization. + +You can see this pattern in practice in projects like [Nx](https://github.com/elixir-nx/nx) and [DNS Cluster](https://github.com/phoenixframework/dns_cluster). These libraries require that you list processes under your own supervision tree: + +```elixir +children = [ + {DNSCluster, query: "my.subdomain"} +] +``` + +In such cases, if the users of `DNSCluster` need to configure DNSCluster per environment, they can be the ones reading from the application environment, without the library forcing them to: + +```elixir +children = [ + {DNSCluster, query: Application.get_env(:my_app, :dns_cluster_query) || :ignore} +] +``` + +Some libraries, such as [Ecto](https://github.com/elixir-ecto/ecto), allow you to pass your application name as an option (called `:otp_app` or similar) and then automatically read the environment from *your* application. While this addresses the issue with the application environment being global, as they read from each individual application, it comes at the cost of some indirection, compared to the example above where users explicitly read their application environment from their own code, whenever desired. + +#### Additional remarks: Compile-time configuration + +A similar discussion entails compile-time configuration. What if a library author requires some configuration to be provided at compilation time? + +Once again, instead of forcing users of your library to provide compile-time configuration, you may want to allow users of your library to generate the code themselves. That's the approach taken by libraries such as [Ecto](https://github.com/elixir-ecto/ecto): + +```elixir +defmodule MyApp.Repo do + use Ecto.Repo, adapter: Ecto.Adapters.Postgres +end +``` + +Instead of forcing developers to share a single repository, Ecto allows its users to define as many repositories as they want. Given the `:adapter` configuration is required at compile-time, it is a required value on `use Ecto.Repo`. If developers want to configure the adapter per environment, then it is their choice: + +```elixir +defmodule MyApp.Repo do + use Ecto.Repo, adapter: Application.compile_env(:my_app, :repo_adapter) +end +``` + +On the other hand, [code generation comes with its own anti-patterns](macro-anti-patterns.md), and must be considered carefully. That's to say: while using the application environment for libraries is discouraged, especially compile-time configuration, in some cases they may be the best option. For example, consider a library needs to parse CSV or JSON files to generate code based on data files. In such cases, it is best to provide reasonable defaults and make them customizable via the application environment, instead of asking each user of your library to generate the exact same code. + +#### Additional remarks: Mix tasks For Mix tasks and related tools, it may be necessary to provide per-project configuration. For example, imagine you have a `:linter` project, which supports setting the output file and the verbosity level. You may choose to configure it through application environment: From ba8fb4dff1dce282489d49e7554f5bd4a1043ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 28 Dec 2023 17:11:12 +0100 Subject: [PATCH 114/185] Improve yecc/leex warnings, closes #13213 --- lib/mix/lib/mix/tasks/compile.leex.ex | 2 +- lib/mix/lib/mix/tasks/compile.yecc.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/lib/mix/tasks/compile.leex.ex b/lib/mix/lib/mix/tasks/compile.leex.ex index 145c54d1c21..3594e060980 100644 --- a/lib/mix/lib/mix/tasks/compile.leex.ex +++ b/lib/mix/lib/mix/tasks/compile.leex.ex @@ -67,7 +67,7 @@ defmodule Mix.Tasks.Compile.Leex do # TODO: Remove me in Elixir v2.0 unless :leex in List.wrap(project[:compilers]) do IO.warn( - "in order to compile .xrl files, you must add \"compilers: [:leex] ++ Mix.compilers()\" to the \"def project\" section of your mix.exs" + "in order to compile .xrl files, you must add \"compilers: [:leex] ++ Mix.compilers()\" to the \"def project\" section of #{project[:app]}'s mix.exs" ) end diff --git a/lib/mix/lib/mix/tasks/compile.yecc.ex b/lib/mix/lib/mix/tasks/compile.yecc.ex index ec3420e7db5..22ad67f1df5 100644 --- a/lib/mix/lib/mix/tasks/compile.yecc.ex +++ b/lib/mix/lib/mix/tasks/compile.yecc.ex @@ -68,7 +68,7 @@ defmodule Mix.Tasks.Compile.Yecc do # TODO: Remove me in Elixir v2.0 unless :yecc in List.wrap(project[:compilers]) do IO.warn( - "in order to compile .yrl files, you must add \"compilers: [:yecc] ++ Mix.compilers()\" to the \"def project\" section of your mix.exs" + "in order to compile .yrl files, you must add \"compilers: [:yecc] ++ Mix.compilers()\" to the \"def project\" section of #{project[:app]}'s mix.exs" ) end From 9ae7c39125561ebd8e56b2421698443bc952cb1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 31 Dec 2023 18:20:20 +0100 Subject: [PATCH 115/185] Update CHANGELOG, closes #13217 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19ff0980edf..29fd5111f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,6 +118,7 @@ Finally, we have started enriching our documentation with [Mermaid.js](https://m * [mix compile.elixir] Pass original exception down to diagnostic `:details` when possible * [mix compile.elixir] Optimize scenario where there are thousands of files in `lib/` and one of them is changed * [mix deps.clean] Emit a warning instead of crashing when a dependency cannot be removed + * [mix escript.build] Escripts now strip .beam files by default, which leads to smaller escripts. However, if you are using escripts to access Elixir docs or compile Elixir code, documentation and deprecation metadata is no longer available. Set `strip_beams: false` in your escript configuration in your `mix.exs` to keep all metadata * [mix escript.install] Support `--sparse` option * [mix release] Include `include/` directory in releases * [mix test] Allow testing multiple file:line at once, such as `mix test test/foo_test.exs:13 test/bar_test.exs:27` From 0d671dafea47bbe789ffa1ecf58275a38bd845f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 2 Jan 2024 16:42:05 +0100 Subject: [PATCH 116/185] Improve anti-pattern titles --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 2 +- lib/elixir/pages/anti-patterns/macro-anti-patterns.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index d97a2534b9e..0d31fbf9191 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -2,7 +2,7 @@ This document outlines potential anti-patterns related to your code and particular Elixir idioms and features. -## Comments +## Comments overuse #### Problem diff --git a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md index b8be9d93672..4693e91c44e 100644 --- a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md @@ -2,7 +2,7 @@ This document outlines potential anti-patterns related to meta-programming. -## Large code generation by macros +## Large code generation #### Problem From 197351dd7350ebe5ea539d4c566f9cc229b114a3 Mon Sep 17 00:00:00 2001 From: "Ryan B. Harvey" Date: Tue, 2 Jan 2024 11:03:37 -0600 Subject: [PATCH 117/185] Fix a comment typo in the Code Anti-Patterns doc page (#13219) --- lib/elixir/pages/anti-patterns/code-anti-patterns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md index 0d31fbf9191..77ff8476204 100644 --- a/lib/elixir/pages/anti-patterns/code-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -372,7 +372,7 @@ iex> Graphics.plot(point_2d) {2, 3, nil} iex> Graphics.plot(bad_point) ** (KeyError) key :x not found in: %{y: 3, z: 4} # <= explicitly warns that - graphic.ex:4: Graphics.plot/1 # <= the :z key does not exist! + graphic.ex:4: Graphics.plot/1 # <= the :x key does not exist! ``` Overall, the usage of `map.key` and `map[:key]` encode important information about your data structure, allowing developers to be clear about their intent. See both `Map` and `Access` module documentation for more information and examples. From 1ece71a831f5fc87ba7895feb9316394ac64601d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 7 Jan 2024 23:07:04 +0100 Subject: [PATCH 118/185] Handle Windows separators on mix test (#13232) Closes #13225. --- lib/ex_unit/lib/ex_unit/filters.ex | 2 +- lib/ex_unit/test/ex_unit/filters_test.exs | 45 +++++++++++++---------- lib/mix/test/mix/tasks/test_test.exs | 22 ++++++++++- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/filters.ex b/lib/ex_unit/lib/ex_unit/filters.ex index 9b3f459802a..1f51908249d 100644 --- a/lib/ex_unit/lib/ex_unit/filters.ex +++ b/lib/ex_unit/lib/ex_unit/filters.ex @@ -51,7 +51,7 @@ defmodule ExUnit.Filters do [path | parts] -> {path_parts, line_numbers} = Enum.split_while(parts, &(to_line_number(&1) == nil)) - path = Enum.join([path | path_parts], ":") + path = Enum.join([path | path_parts], ":") |> Path.split() |> Path.join() lines = for n <- line_numbers, valid_number = validate_line_number(n), do: valid_number case lines do diff --git a/lib/ex_unit/test/ex_unit/filters_test.exs b/lib/ex_unit/test/ex_unit/filters_test.exs index 44df58a7baa..0f0510f6470 100644 --- a/lib/ex_unit/test/ex_unit/filters_test.exs +++ b/lib/ex_unit/test/ex_unit/filters_test.exs @@ -198,28 +198,33 @@ defmodule ExUnit.FiltersTest do windows_path = "C:\\some\\path.exs" for path <- [unix_path, windows_path] do + fixed_path = path |> Path.split() |> Path.join() + assert ExUnit.Filters.parse_path("#{path}:123") == - {path, [exclude: [:test], include: [location: {path, 123}]]} + {fixed_path, [exclude: [:test], include: [location: {fixed_path, 123}]]} - assert ExUnit.Filters.parse_path(path) == {path, []} + assert ExUnit.Filters.parse_path(path) == {fixed_path, []} assert ExUnit.Filters.parse_path("#{path}:123notreallyalinenumber123") == - {"#{path}:123notreallyalinenumber123", []} + {"#{fixed_path}:123notreallyalinenumber123", []} assert ExUnit.Filters.parse_path("#{path}:123:456") == - {path, [exclude: [:test], include: [location: {path, [123, 456]}]]} + {fixed_path, [exclude: [:test], include: [location: {fixed_path, [123, 456]}]]} assert ExUnit.Filters.parse_path("#{path}:123notalinenumber123:456") == - {"#{path}:123notalinenumber123", - [exclude: [:test], include: [location: {"#{path}:123notalinenumber123", 456}]]} + {"#{fixed_path}:123notalinenumber123", + [ + exclude: [:test], + include: [location: {"#{fixed_path}:123notalinenumber123", 456}] + ]} output = ExUnit.CaptureIO.capture_io(:stderr, fn -> assert ExUnit.Filters.parse_path("#{path}:123:456notalinenumber456") == - {path, [{:exclude, [:test]}, {:include, [location: {path, 123}]}]} + {fixed_path, [{:exclude, [:test]}, {:include, [location: {fixed_path, 123}]}]} assert ExUnit.Filters.parse_path("#{path}:123:0:-789:456") == - {path, [exclude: [:test], include: [location: {path, [123, 456]}]]} + {fixed_path, [exclude: [:test], include: [location: {fixed_path, [123, 456]}]]} end) assert output =~ "invalid line number given as ExUnit filter: 456notalinenumber456" @@ -231,25 +236,25 @@ defmodule ExUnit.FiltersTest do test "multiple file paths with line numbers" do unix_path = "test/some/path.exs" windows_path = "C:\\some\\path.exs" - other_unix_path = "test/some/other_path.exs" + other_unix_path = "test//some//other_path.exs" other_windows_path = "C:\\some\\other_path.exs" - for {path, other_path} <- [ - {unix_path, other_unix_path}, - {windows_path, other_windows_path} - ] do + for {path, other_path} <- [{unix_path, other_unix_path}, {windows_path, other_windows_path}] do + fixed_path = path |> Path.split() |> Path.join() + fixed_other_path = other_path |> Path.split() |> Path.join() + assert ExUnit.Filters.parse_paths([path, "#{other_path}:456:789"]) == - {[path, other_path], + {[fixed_path, fixed_other_path], [ exclude: [:test], - include: [location: {other_path, [456, 789]}] + include: [location: {fixed_other_path, [456, 789]}] ]} assert ExUnit.Filters.parse_paths(["#{path}:123", "#{other_path}:456"]) == - {[path, other_path], + {[fixed_path, fixed_other_path], [ exclude: [:test], - include: [location: {path, 123}, location: {other_path, 456}] + include: [location: {fixed_path, 123}, location: {fixed_other_path, 456}] ]} output = @@ -258,12 +263,12 @@ defmodule ExUnit.FiltersTest do "#{path}:123:0:-789:456", "#{other_path}:321:0:-987:654" ]) == - {[path, other_path], + {[fixed_path, fixed_other_path], [ exclude: [:test], include: [ - location: {path, [123, 456]}, - location: {other_path, [321, 654]} + location: {fixed_path, [123, 456]}, + location: {fixed_other_path, [321, 654]} ] ]} end) diff --git a/lib/mix/test/mix/tasks/test_test.exs b/lib/mix/test/mix/tasks/test_test.exs index b3b4a686401..f0809cd7b96 100644 --- a/lib/mix/test/mix/tasks/test_test.exs +++ b/lib/mix/test/mix/tasks/test_test.exs @@ -223,6 +223,15 @@ defmodule Mix.Tasks.TestTest do output = mix(["test", "test/passing_and_failing_test_failed.exs", "--failed"]) assert output =~ "1 test, 1 failure" + # Plus line + output = mix(["test", "test/passing_and_failing_test_failed.exs:5", "--failed"]) + assert output =~ "1 test, 1 failure" + + if windows?() do + output = mix(["test", "test\\passing_and_failing_test_failed.exs:5", "--failed"]) + assert output =~ "1 test, 1 failure" + end + # `--failed` composes with an `--only` filter by running the intersection. # Of the failing tests, 1 is tagged with `@tag :foo`. # Of the passing tests, 1 is tagged with `@tag :foo`. @@ -509,17 +518,28 @@ defmodule Mix.Tasks.TestTest do refute output =~ "==> foo" refute output =~ "Paths given to \"mix test\" did not match any directory/file" - output = mix(["test", "apps/foo/test/foo_tests.exs:9", "apps/bar/test/bar_tests.exs:5"]) + casing = + if windows?() do + "apps\\bar\\test\\bar_tests.exs:5" + else + "apps/bar/test/bar_tests.exs:5" + end + + output = mix(["test", "apps/foo/test/foo_tests.exs:9", casing]) assert output =~ """ Excluding tags: [:test] Including tags: [location: {"test/foo_tests.exs", 9}] """ + assert output =~ "1 test, 0 failures\n" + assert output =~ """ Excluding tags: [:test] Including tags: [location: {"test/bar_tests.exs", 5}] """ + + assert output =~ "4 tests, 0 failures, 3 excluded\n" end) end end From 6f5715fc30c01f6b42f26e063c9b1a7c58e9f385 Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Mon, 8 Jan 2024 18:05:11 +0900 Subject: [PATCH 119/185] Replace single quotes in charlist in doc (#13233) --- lib/elixir/pages/references/patterns-and-guards.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/elixir/pages/references/patterns-and-guards.md b/lib/elixir/pages/references/patterns-and-guards.md index dd53d1b5da3..f5728f5c6ed 100644 --- a/lib/elixir/pages/references/patterns-and-guards.md +++ b/lib/elixir/pages/references/patterns-and-guards.md @@ -159,13 +159,13 @@ iex> [head | tail] = [] Given charlists are represented as a list of integers, one can also perform prefix matches on charlists using the list concatenation operator ([`++`](`++/2`)): ```elixir -iex> 'hello ' ++ world = 'hello world' -'hello world' +iex> ~c"hello " ++ world = ~c"hello world" +~c"hello world" iex> world -'world' +~c"world" ``` -Which is equivalent to matching on `[?h, ?e, ?l, ?l, ?o, ?\s | world]`. Suffix matches (`hello ++ ' world'`) are not valid patterns. +Which is equivalent to matching on `[?h, ?e, ?l, ?l, ?o, ?\s | world]`. Suffix matches (`hello ++ ~c" world"`) are not valid patterns. ### Maps From f2dae095f850733a66fcbdc14429081e7e7ca663 Mon Sep 17 00:00:00 2001 From: Will Douglas Date: Fri, 29 Dec 2023 11:55:16 -0300 Subject: [PATCH 120/185] Fix typo in docs: for -> force (#13215) Co-authored-by: Will Douglas --- .../pages/getting-started/binaries-strings-and-charlists.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/getting-started/binaries-strings-and-charlists.md b/lib/elixir/pages/getting-started/binaries-strings-and-charlists.md index d3e5940b44b..abc59774bfc 100644 --- a/lib/elixir/pages/getting-started/binaries-strings-and-charlists.md +++ b/lib/elixir/pages/getting-started/binaries-strings-and-charlists.md @@ -257,7 +257,7 @@ iex> heartbeats_per_minute = [99, 97, 116] ~c"cat" ``` -You can always for charlists to be printed in their list representation by calling the `inspect/2` function: +You can always force charlists to be printed in their list representation by calling the `inspect/2` function: ```elixir iex> inspect(heartbeats_per_minute, charlists: :as_list) From 6e295998effdf7e32896d9e39847d2eebcb7582e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Sun, 24 Dec 2023 20:40:13 -0300 Subject: [PATCH 121/185] Fix typo in docs: `Keywoird` -> `Keyword` (#13205) --- lib/elixir/pages/anti-patterns/process-anti-patterns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md index 6c513b81678..8025437806c 100644 --- a/lib/elixir/pages/anti-patterns/process-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -263,7 +263,7 @@ defmodule Counter do @doc "Starts a counter process." def start_link(opts \\ []) do initial_valye = Keyword.get(opts, :initial_value, 0) - name = Keywoird.get(opts, :name, __MODULE__) + name = Keyword.get(opts, :name, __MODULE__) GenServer.start(__MODULE__, initial_value, name: name) end From 4de331597acca51a782709696646c1d7c9d8eb13 Mon Sep 17 00:00:00 2001 From: Adebisi Adeyeye <68188123+badeyeye1@users.noreply.github.com> Date: Thu, 28 Dec 2023 01:00:13 +0100 Subject: [PATCH 122/185] Fix typo in docs: initial_valye -> initial_value (#13211) Co-authored-by: Adebisi Adeyeye --- lib/elixir/pages/anti-patterns/process-anti-patterns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md index 8025437806c..e3258213bf7 100644 --- a/lib/elixir/pages/anti-patterns/process-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -262,7 +262,7 @@ defmodule Counter do @doc "Starts a counter process." def start_link(opts \\ []) do - initial_valye = Keyword.get(opts, :initial_value, 0) + initial_value = Keyword.get(opts, :initial_value, 0) name = Keyword.get(opts, :name, __MODULE__) GenServer.start(__MODULE__, initial_value, name: name) end From f83b4e4e8400a6d46b162e72fb7d8b898b2c4ceb Mon Sep 17 00:00:00 2001 From: Hussien Liban Date: Wed, 27 Dec 2023 14:56:30 +0300 Subject: [PATCH 123/185] Update design-anti-patterns.md (#13210) Not setting an option returns just the integer --- lib/elixir/pages/anti-patterns/design-anti-patterns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index fd1aa23a987..22407c87472 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -30,7 +30,7 @@ end ```elixir iex> AlternativeInteger.parse("13") -{13, ""} +13 iex> AlternativeInteger.parse("13", discard_rest: true) 13 iex> AlternativeInteger.parse("13", discard_rest: false) From 17b9ba61d90618dfc1824224d3e787fe44df8cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 10 Jan 2024 10:16:49 +0100 Subject: [PATCH 124/185] Do not crash parallel compiler on external reports, closes #13224 --- lib/elixir/lib/kernel/parallel_compiler.ex | 1 - .../elixir/kernel/parallel_compiler_test.exs | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index d88b23dc730..f806356883f 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -455,7 +455,6 @@ defmodule Kernel.ParallelCompiler do # No more queue, nothing waiting, this cycle is done defp spawn_workers([], spawned, waiting, files, result, warnings, errors, state) when map_size(spawned) == 0 and map_size(waiting) == 0 do - [] = errors [] = files cycle_return = each_cycle_return(state.each_cycle.()) state = cycle_timing(result, state) diff --git a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs index 9edbe5c0fc1..a9d2a83a6ae 100644 --- a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs +++ b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs @@ -196,6 +196,28 @@ defmodule Kernel.ParallelCompilerTest do purge([QuickExample]) end + test "does not crash on external reports" do + [fixture] = + write_tmp( + "compile_quoted", + quick_example: """ + defmodule CompileQuoted do + try do + Code.compile_quoted({:fn, [], [{:->, [], [[], quote(do: unknown_var)]}]}) + rescue + _ -> :ok + end + end + """ + ) + + assert capture_io(:stderr, fn -> + assert {:ok, [CompileQuoted], []} = Kernel.ParallelCompiler.compile([fixture]) + end) =~ "undefined variable \"unknown_var\"" + after + purge([CompileQuoted]) + end + test "does not hang on missing dependencies" do [fixture] = write_tmp( From b96211a325274f48dbf8278405f67fa11bae9775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 20 Nov 2023 21:19:40 +0800 Subject: [PATCH 125/185] Do not assume there is a $HOME, closes #13127 --- lib/iex/lib/iex/evaluator.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/iex/lib/iex/evaluator.ex b/lib/iex/lib/iex/evaluator.ex index 5887625ec7c..3086e4ae3dc 100644 --- a/lib/iex/lib/iex/evaluator.ex +++ b/lib/iex/lib/iex/evaluator.ex @@ -253,9 +253,10 @@ defmodule IEx.Evaluator do if path do [path] else - Enum.map([".", System.get_env("IEX_HOME", "~")], fn dir -> - dir |> Path.join(".iex.exs") |> Path.expand() - end) + # Do not assume there is a $HOME + for dir <- [".", System.get_env("IEX_HOME") || System.user_home()], + dir != nil, + do: dir |> Path.join(".iex.exs") |> Path.expand() end path = Enum.find(candidates, &File.regular?/1) From 691d402341a2bfb47419f710c40bc01698a689eb Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Sat, 13 Jan 2024 17:58:45 +0900 Subject: [PATCH 126/185] Fix Code.Normalizer for keyword operand with :do key (#13250) --- lib/elixir/lib/code/normalizer.ex | 10 +++++----- .../test/elixir/code_normalizer/quoted_ast_test.exs | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index 5a1c3c2fe68..7abded155b3 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -352,6 +352,10 @@ defmodule Code.Normalizer do last = List.last(args) cond do + not allow_keyword?(form, arity) -> + args = normalize_args(args, %{state | parent_meta: meta}) + {form, meta, args} + Keyword.has_key?(meta, :do) or match?([{{:__block__, _, [:do]}, _} | _], last) -> # def foo do :ok end # def foo, do: :ok @@ -363,7 +367,7 @@ defmodule Code.Normalizer do meta = meta ++ [do: [line: line], end: [line: line]] normalize_kw_blocks(form, meta, args, state) - allow_keyword?(form, arity) -> + true -> args = normalize_args(args, %{state | parent_meta: meta}) {last_arg, leading_args} = List.pop_at(args, -1, []) @@ -384,10 +388,6 @@ defmodule Code.Normalizer do end {form, meta, leading_args ++ last_args} - - true -> - args = normalize_args(args, %{state | parent_meta: meta}) - {form, meta, args} end end diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index f70cd98f525..f6deeffb61f 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -539,6 +539,10 @@ defmodule Code.Normalizer.QuotedASTTest do "\e[34m[\e[0m\e[32ma:\e[0m \e[33m1\e[0m, \e[32mb:\e[0m \e[33m2\e[0m\e[34m]\e[0m" end + test "keyword list with :do as operand" do + assert quoted_to_string(quote(do: a = [do: 1])) == "a = [do: 1]" + end + test "interpolation" do assert quoted_to_string(quote(do: "foo#{bar}baz")) == ~S["foo#{bar}baz"] end From 6b69c7f5acee418e986d7e1d3efbe7df05d86ff6 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 15 Jan 2024 13:42:31 +0100 Subject: [PATCH 127/185] Fix the explanation to match the explained code example in docs (#13255) The description incorrectly states that both processes are initialized with 0, while in the code the second process receives a non-default initial value. --- lib/elixir/pages/anti-patterns/process-anti-patterns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md index e3258213bf7..ad13b623cc5 100644 --- a/lib/elixir/pages/anti-patterns/process-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -310,7 +310,7 @@ iex> Counter.bump(Counter, 7) #### Refactoring -To ensure that clients of a library have full control over their systems, regardless of the number of processes used and the lifetime of each one, all processes must be started inside a supervision tree. As shown below, this code uses a `Supervisor` as a supervision tree. When this Elixir application is started, two different counters (`Counter` and `:other_counter`) are also started as child processes of the `Supervisor` named `App.Supervisor`. Both are initialized to `0`. By means of this supervision tree, it is possible to manage the lifecycle of all child processes (stopping or restarting each one), improving the visibility of the entire app. +To ensure that clients of a library have full control over their systems, regardless of the number of processes used and the lifetime of each one, all processes must be started inside a supervision tree. As shown below, this code uses a `Supervisor` as a supervision tree. When this Elixir application is started, two different counters (`Counter` and `:other_counter`) are also started as child processes of the `Supervisor` named `App.Supervisor`. One is initialized with `0`, the other with `15`. By means of this supervision tree, it is possible to manage the lifecycle of all child processes (stopping or restarting each one), improving the visibility of the entire app. ```elixir defmodule SupervisedProcess.Application do From 7d100fff9f11d32c8a318f638453f1d8208e0de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 17 Jan 2024 23:11:40 +0100 Subject: [PATCH 128/185] Always log errors at the end of compilation --- lib/elixir/lib/kernel/parallel_compiler.ex | 43 +++++++++++++------ lib/elixir/lib/module/parallel_checker.ex | 10 ++--- lib/elixir/src/elixir_errors.erl | 17 +++++--- .../elixir/kernel/parallel_compiler_test.exs | 4 +- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index f806356883f..e148af18862 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -455,6 +455,11 @@ defmodule Kernel.ParallelCompiler do # No more queue, nothing waiting, this cycle is done defp spawn_workers([], spawned, waiting, files, result, warnings, errors, state) when map_size(spawned) == 0 and map_size(waiting) == 0 do + # Print any spurious error that we may have found + Enum.map(errors, fn {diagnostic, read_snippet} -> + :elixir_errors.print_diagnostic(diagnostic, read_snippet) + end) + [] = files cycle_return = each_cycle_return(state.each_cycle.()) state = cycle_timing(result, state) @@ -509,8 +514,9 @@ defmodule Kernel.ParallelCompiler do if deadlocked do spawn_workers(deadlocked, spawned, waiting, files, result, warnings, errors, state) else - deadlock_errors = handle_deadlock(waiting, files) - {return_error(deadlock_errors ++ errors, warnings), state} + return_error(warnings, errors, state, fn -> + handle_deadlock(waiting, files) + end) end end @@ -680,12 +686,13 @@ defmodule Kernel.ParallelCompiler do state = %{state | timer_ref: timer_ref} spawn_workers(queue, spawned, waiting, files, result, warnings, errors, state) - {:diagnostic, %{severity: :warning} = diagnostic} -> - warnings = [Module.ParallelChecker.format_diagnostic_file(diagnostic) | warnings] + {:diagnostic, %{severity: :warning, file: file} = diagnostic, read_snippet} -> + :elixir_errors.print_diagnostic(diagnostic, read_snippet) + warnings = [%{diagnostic | file: file && Path.absname(file)} | warnings] wait_for_messages(queue, spawned, waiting, files, result, warnings, errors, state) - {:diagnostic, %{severity: :error} = diagnostic} -> - errors = [Module.ParallelChecker.format_diagnostic_file(diagnostic) | errors] + {:diagnostic, %{severity: :error} = diagnostic, read_snippet} -> + errors = [{diagnostic, read_snippet} | errors] wait_for_messages(queue, spawned, waiting, files, result, warnings, errors, state) {:file_ok, child_pid, ref, file, lexical} -> @@ -705,10 +712,13 @@ defmodule Kernel.ParallelCompiler do spawn_workers(queue, new_spawned, waiting, new_files, result, warnings, errors, state) {:file_error, child_pid, file, {kind, reason, stack}} -> - print_error(file, kind, reason, stack) {_file, _new_spawned, new_files} = discard_file_pid(spawned, files, child_pid) terminate(new_files) - {return_error([to_error(file, kind, reason, stack) | errors], warnings), state} + + return_error(warnings, errors, state, fn -> + print_error(file, kind, reason, stack) + [to_error(file, kind, reason, stack)] + end) {:DOWN, ref, :process, pid, reason} when is_map_key(spawned, ref) -> # async spawned processes have no file, so we always have to delete the ref directly @@ -717,18 +727,27 @@ defmodule Kernel.ParallelCompiler do {file, spawned, files} = discard_file_pid(spawned, files, pid) if file do - print_error(file.file, :exit, reason, []) terminate(files) - {return_error([to_error(file.file, :exit, reason, []) | errors], warnings), state} + + return_error(warnings, errors, state, fn -> + print_error(file.file, :exit, reason, []) + [to_error(file.file, :exit, reason, [])] + end) else wait_for_messages(queue, spawned, waiting, files, result, warnings, errors, state) end end end - defp return_error(errors, warnings) do + defp return_error(warnings, errors, state, fun) do + errors = + Enum.map(errors, fn {%{file: file} = diagnostic, read_snippet} -> + :elixir_errors.print_diagnostic(diagnostic, read_snippet) + %{diagnostic | file: file && Path.absname(file)} + end) + info = %{compile_warnings: Enum.reverse(warnings), runtime_warnings: []} - {:error, Enum.reverse(errors), info} + {{:error, Enum.reverse(errors, fun.()), info}, state} end defp update_result(result, kind, module, value) do diff --git a/lib/elixir/lib/module/parallel_checker.ex b/lib/elixir/lib/module/parallel_checker.ex index d4e51abb9e5..9b40b4ccf53 100644 --- a/lib/elixir/lib/module/parallel_checker.ex +++ b/lib/elixir/lib/module/parallel_checker.ex @@ -167,8 +167,9 @@ defmodule Module.ParallelChecker do defp collect_results(count, diagnostics) do receive do - {:diagnostic, diagnostic} -> - diagnostic = format_diagnostic_file(diagnostic) + {:diagnostic, %{file: file} = diagnostic, read_snippet} -> + :elixir_errors.print_diagnostic(diagnostic, read_snippet) + diagnostic = %{diagnostic | file: file && Path.absname(file)} collect_results(count, [diagnostic | diagnostics]) {__MODULE__, _module, new_diagnostics} -> @@ -287,11 +288,6 @@ defmodule Module.ParallelChecker do end end - @doc false - def format_diagnostic_file(%{file: file} = diagnostic) do - %{diagnostic | file: file && Path.absname(file)} - end - ## Warning helpers defp group_warnings(warnings) do diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 96da99388e6..f39f7b6e65b 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -115,14 +115,17 @@ emit_diagnostic(Severity, Position, File, Message, Stacktrace, Options) -> }, case get(elixir_code_diagnostics) of - undefined -> print_diagnostic(Diagnostic, ReadSnippet); - {Tail, true} -> put(elixir_code_diagnostics, {[print_diagnostic(Diagnostic, ReadSnippet) | Tail], true}); - {Tail, false} -> put(elixir_code_diagnostics, {[Diagnostic | Tail], false}) - end, + undefined -> + case get(elixir_compiler_info) of + undefined -> print_diagnostic(Diagnostic, ReadSnippet); + {CompilerPid, _} -> CompilerPid ! {diagnostic, Diagnostic, ReadSnippet} + end; + + {Tail, true} -> + put(elixir_code_diagnostics, {[print_diagnostic(Diagnostic, ReadSnippet) | Tail], true}); - case get(elixir_compiler_info) of - undefined -> ok; - {CompilerPid, _} -> CompilerPid ! {diagnostic, Diagnostic} + {Tail, false} -> + put(elixir_code_diagnostics, {[Diagnostic | Tail], false}) end, ok. diff --git a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs index a9d2a83a6ae..fa419681609 100644 --- a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs +++ b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs @@ -327,7 +327,7 @@ defmodule Kernel.ParallelCompilerTest do msg = capture_io(:stderr, fn -> fixtures = [foo, bar] - assert {:error, [foo_error, bar_error], []} = Kernel.ParallelCompiler.compile(fixtures) + assert {:error, [bar_error, foo_error], []} = Kernel.ParallelCompiler.compile(fixtures) assert bar_error == {bar, nil, "deadlocked waiting on module FooDeadlock"} assert foo_error == {foo, nil, "deadlocked waiting on module BarDeadlock"} end) @@ -415,7 +415,7 @@ defmodule Kernel.ParallelCompilerTest do capture_io(:stderr, fn -> fixtures = [foo, bar] - assert {:error, [foo_error, bar_error], []} = Kernel.ParallelCompiler.compile(fixtures) + assert {:error, [bar_error, foo_error], []} = Kernel.ParallelCompiler.compile(fixtures) assert {^bar, nil, "deadlocked waiting on module FooAsyncDeadlock"} = bar_error assert {^foo, nil, "deadlocked waiting on pmap [#PID<" <> _} = foo_error end) From 9ea950c9afcfe8fff93e08def4bf6bc3cf7a4de0 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 16 Jan 2024 21:15:26 +0100 Subject: [PATCH 129/185] Macro anti-patterns: fix incorrect error message in code examples (#13259) --- lib/elixir/pages/anti-patterns/macro-anti-patterns.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md index 4693e91c44e..1ca5362ce3f 100644 --- a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md @@ -24,7 +24,7 @@ defmodule Routes do end if not is_atom(handler) do - raise ArgumentError, "route must be a module" + raise ArgumentError, "handler must be a module" end @store_route_for_compilation {route, handler} @@ -51,7 +51,7 @@ defmodule Routes do end if not is_atom(handler) do - raise ArgumentError, "route must be a module" + raise ArgumentError, "handler must be a module" end Module.put_attribute(module, :store_route_for_compilation, {route, handler}) From 6dccefe687b268bbd3e4b06a6b8e85f1540f349b Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 13 Jan 2024 11:42:44 -0500 Subject: [PATCH 130/185] Fix :from_interpolation docs (#13251) --- lib/elixir/lib/macro.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/lib/macro.ex b/lib/elixir/lib/macro.ex index 844ed552010..b6a9c530691 100644 --- a/lib/elixir/lib/macro.ex +++ b/lib/elixir/lib/macro.ex @@ -115,7 +115,7 @@ defmodule Macro do * `:from_brackets` - Used to determine whether a call to `Access.get/3` is from bracket syntax. - * `:from_interpolation` - Used to determine whether a call to `Access.get/3` is + * `:from_interpolation` - Used to determine whether a call to `Kernel.to_string/1` is from interpolation. * `:generated` - Whether the code should be considered as generated by From 111b48dcf44b2114dbab3fa10f25f1053a6dbe06 Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Thu, 18 Jan 2024 08:06:35 +0900 Subject: [PATCH 131/185] Resolve relative paths in exunit filter (#13258) --- lib/ex_unit/lib/ex_unit/filters.ex | 2 +- lib/ex_unit/test/ex_unit/filters_test.exs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/filters.ex b/lib/ex_unit/lib/ex_unit/filters.ex index 1f51908249d..82f509c6c7c 100644 --- a/lib/ex_unit/lib/ex_unit/filters.ex +++ b/lib/ex_unit/lib/ex_unit/filters.ex @@ -45,7 +45,7 @@ defmodule ExUnit.Filters do end defp extract_line_numbers(file_path) do - case String.split(file_path, ":") do + case Path.relative_to_cwd(file_path) |> String.split(":") do [path] -> {path, []} diff --git a/lib/ex_unit/test/ex_unit/filters_test.exs b/lib/ex_unit/test/ex_unit/filters_test.exs index 0f0510f6470..a11947df37d 100644 --- a/lib/ex_unit/test/ex_unit/filters_test.exs +++ b/lib/ex_unit/test/ex_unit/filters_test.exs @@ -196,9 +196,10 @@ defmodule ExUnit.FiltersTest do test "file paths with line numbers" do unix_path = "test/some/path.exs" windows_path = "C:\\some\\path.exs" + unix_path_with_dot = "./test/some/path.exs" - for path <- [unix_path, windows_path] do - fixed_path = path |> Path.split() |> Path.join() + for path <- [unix_path, windows_path, unix_path_with_dot] do + fixed_path = path |> Path.split() |> Path.join() |> Path.relative_to_cwd() assert ExUnit.Filters.parse_path("#{path}:123") == {fixed_path, [exclude: [:test], include: [location: {fixed_path, 123}]]} From bb8ad4406b1499f8604f5ade7a049bf4056901e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sun, 14 Jan 2024 16:22:38 +0100 Subject: [PATCH 132/185] Improve ast metadata docs --- lib/elixir/lib/macro.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/elixir/lib/macro.ex b/lib/elixir/lib/macro.ex index b6a9c530691..e7fabc0e22c 100644 --- a/lib/elixir/lib/macro.ex +++ b/lib/elixir/lib/macro.ex @@ -136,8 +136,9 @@ defmodule Macro do * `:closing` - contains metadata about the closing pair, such as a `}` in a tuple or in a map, or such as the closing `)` in a function call - with parens. The `:closing` does not delimit the end of expression if - there are `:do` and `:end` metadata (when `:token_metadata` is true) + with parens (when `:token_metadata` is true). If the function call + has a do-end block attached to it, its metadata is found under the + `:do` and `:end` metadata * `:column` - the column number of the AST node (when `:columns` is true). Note column information is always discarded from quoted code. @@ -154,8 +155,8 @@ defmodule Macro do `do`-`end` blocks (when `:token_metadata` is true) * `:end_of_expression` - denotes when the end of expression effectively - happens. Available for all expressions except the last one inside a - `__block__` (when `:token_metadata` is true) + happens (when `:token_metadata` is true). Available for all expressions + except the last one inside a `__block__` * `:indentation` - indentation of a sigil heredoc From c35edd1ee881f2c3cc629f8c1f1cfe38d252b324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 17 Jan 2024 23:14:14 +0100 Subject: [PATCH 133/185] Improve end_of_expression docs --- lib/elixir/lib/macro.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/macro.ex b/lib/elixir/lib/macro.ex index e7fabc0e22c..66312f1c6cf 100644 --- a/lib/elixir/lib/macro.ex +++ b/lib/elixir/lib/macro.ex @@ -155,8 +155,10 @@ defmodule Macro do `do`-`end` blocks (when `:token_metadata` is true) * `:end_of_expression` - denotes when the end of expression effectively - happens (when `:token_metadata` is true). Available for all expressions - except the last one inside a `__block__` + happens (when `:token_metadata` is true). This is only available for + direct children of a `__block__`, and it is either the location of a + newline or of the `;` character. The last expression of `__block__` + does not have this metadata. * `:indentation` - indentation of a sigil heredoc From ad778a6f2990ebbc7f1c1a2326524135a6170c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 19 Jan 2024 19:42:55 +0100 Subject: [PATCH 134/185] Fix capitalize for single codepoint (#13268) --- lib/elixir/lib/string.ex | 1 + lib/elixir/test/elixir/string_test.exs | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index 1966c533409..3945f681fbe 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -966,6 +966,7 @@ defmodule String do def capitalize(string, mode) when is_binary(string) do case :unicode_util.gc(string) do + [gc] -> grapheme_to_binary(:string.titlecase([gc])) [gc, rest] -> grapheme_to_binary(:string.titlecase([gc])) <> downcase(rest, mode) [gc | rest] -> grapheme_to_binary(:string.titlecase([gc])) <> downcase(rest, mode) [] -> "" diff --git a/lib/elixir/test/elixir/string_test.exs b/lib/elixir/test/elixir/string_test.exs index 8a411eb78d1..6aa5f622595 100644 --- a/lib/elixir/test/elixir/string_test.exs +++ b/lib/elixir/test/elixir/string_test.exs @@ -254,6 +254,7 @@ defmodule StringTest do test "capitalize/1" do assert String.capitalize("") == "" + assert String.capitalize("1") == "1" assert String.capitalize("abc") == "Abc" assert String.capitalize("ABC") == "Abc" assert String.capitalize("c b a") == "C b a" From da0189e641aed10174a21b430172923d6fde4519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 27 Jan 2024 11:45:59 +0100 Subject: [PATCH 135/185] Fix docs link Related to #13284. --- Makefile | 1 - lib/elixir/lib/enum.ex | 2 +- lib/elixir/pages/anti-patterns/design-anti-patterns.md | 2 +- lib/elixir/pages/getting-started/basic-types.md | 2 +- lib/elixir/pages/getting-started/enumerable-and-streams.md | 2 +- lib/elixir/pages/meta-programming/macros.md | 2 +- lib/elixir/pages/mix-and-otp/dynamic-supervisor.md | 2 +- lib/elixir/pages/references/operators.md | 2 +- lib/elixir/pages/references/patterns-and-guards.md | 2 +- lib/elixir/pages/references/typespecs.md | 2 +- lib/elixir/pages/references/unicode-syntax.md | 2 +- lib/elixir/scripts/elixir_docs.exs | 1 + 12 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index c885b10be4f..d1725414637 100644 --- a/Makefile +++ b/Makefile @@ -189,7 +189,6 @@ docs_elixir: compile ../ex_doc/bin/ex_doc $(Q) rm -rf doc/elixir $(call DOCS_COMPILE,Elixir,elixir,Kernel,--config "lib/elixir/scripts/elixir_docs.exs") $(call DOCS_CONFIG,elixir) - cp -R lib/elixir/pages/images doc/elixir docs_eex: compile ../ex_doc/bin/ex_doc @ echo "==> ex_doc (eex)" diff --git a/lib/elixir/lib/enum.ex b/lib/elixir/lib/enum.ex index e34d665ca4a..373da0a6636 100644 --- a/lib/elixir/lib/enum.ex +++ b/lib/elixir/lib/enum.ex @@ -261,7 +261,7 @@ defmodule Enum do traversed as if it was an enumerable. For a general overview of all functions in the `Enum` module, see - [the `Enum` cheatsheet](enum-cheat.html). + [the `Enum` cheatsheet](enum-cheat.cheatmd). The functions in this module work in linear time. This means that, the time it takes to perform an operation grows at the same rate as the length diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md index 22407c87472..9ca98aeb521 100644 --- a/lib/elixir/pages/anti-patterns/design-anti-patterns.md +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -167,7 +167,7 @@ defmodule MyModule do end ``` -This is only possible because the `File` module provides APIs for reading files with tuples as results (`File.read/1`), as well as a version that raises an exception (`File.read!/1`). The bang (exclamation point) is effectively part of [Elixir's naming conventions](naming-conventions.html#trailing-bang-foo). +This is only possible because the `File` module provides APIs for reading files with tuples as results (`File.read/1`), as well as a version that raises an exception (`File.read!/1`). The bang (exclamation point) is effectively part of [Elixir's naming conventions](naming-conventions.md#trailing-bang-foo). Library authors are encouraged to follow the same practices. In practice, the bang variant is implemented on top of the non-raising version of the code. For example, `File.read!/1` is implemented as: diff --git a/lib/elixir/pages/getting-started/basic-types.md b/lib/elixir/pages/getting-started/basic-types.md index 6b26fba71c7..525d9bb1293 100644 --- a/lib/elixir/pages/getting-started/basic-types.md +++ b/lib/elixir/pages/getting-started/basic-types.md @@ -325,6 +325,6 @@ iex> 1 === 1.0 false ``` -The comparison operators in Elixir can compare across any data type. We say these operators perform _structural comparison_. For more information, you can read our documentation on [Structural vs Semantic comparisons](Kernel.html#module-structural-comparison). +The comparison operators in Elixir can compare across any data type. We say these operators perform _structural comparison_. For more information, you can read our documentation on [Structural vs Semantic comparisons](`Kernel#module-structural-comparison`). Elixir also provides data-types for expressing collections, such as lists and tuples, which we learn next. When we talk about concurrency and fault-tolerance via processes, we will also discuss ports, pids, and references, but that will come on later chapters. Let's move forward. diff --git a/lib/elixir/pages/getting-started/enumerable-and-streams.md b/lib/elixir/pages/getting-started/enumerable-and-streams.md index bc4f81516f0..07526b16292 100644 --- a/lib/elixir/pages/getting-started/enumerable-and-streams.md +++ b/lib/elixir/pages/getting-started/enumerable-and-streams.md @@ -13,7 +13,7 @@ iex> Enum.map(%{1 => 2, 3 => 4}, fn {k, v} -> k * v end) [2, 12] ``` -The `Enum` module provides a huge range of functions to transform, sort, group, filter and retrieve items from enumerables. It is one of the modules developers use frequently in their Elixir code. For a general overview of all functions in the `Enum` module, see [the `Enum` cheatsheet](enum-cheat.html). +The `Enum` module provides a huge range of functions to transform, sort, group, filter and retrieve items from enumerables. It is one of the modules developers use frequently in their Elixir code. For a general overview of all functions in the `Enum` module, see [the `Enum` cheatsheet](enum-cheat.cheatmd). Elixir also provides ranges (see `Range`), which are also enumerable: diff --git a/lib/elixir/pages/meta-programming/macros.md b/lib/elixir/pages/meta-programming/macros.md index ceed3c21531..1f04a344c8c 100644 --- a/lib/elixir/pages/meta-programming/macros.md +++ b/lib/elixir/pages/meta-programming/macros.md @@ -26,7 +26,7 @@ defmodule Unless do end ``` -The function receives the arguments and passes them to `if/2`. However, as we learned in the [previous guide](quote-and-unquote.html), the macro will receive quoted expressions, inject them into the quote, and finally return another quoted expression. +The function receives the arguments and passes them to `if/2`. However, as we learned in the [previous guide](quote-and-unquote.md), the macro will receive quoted expressions, inject them into the quote, and finally return another quoted expression. Let's start `iex` with the module above: diff --git a/lib/elixir/pages/mix-and-otp/dynamic-supervisor.md b/lib/elixir/pages/mix-and-otp/dynamic-supervisor.md index e353133372d..da3789c7d51 100644 --- a/lib/elixir/pages/mix-and-otp/dynamic-supervisor.md +++ b/lib/elixir/pages/mix-and-otp/dynamic-supervisor.md @@ -183,7 +183,7 @@ A GUI should pop up containing all sorts of information about our system, from g In the Applications tab, you will see all applications currently running in your system alongside their supervision tree. You can select the `kv` application to explore it further: -Observer GUI screenshot +Observer GUI screenshot Not only that, as you create new buckets on the terminal, you should see new processes spawned in the supervision tree shown in Observer: diff --git a/lib/elixir/pages/references/operators.md b/lib/elixir/pages/references/operators.md index 41642f0a9f5..73806521124 100644 --- a/lib/elixir/pages/references/operators.md +++ b/lib/elixir/pages/references/operators.md @@ -64,7 +64,7 @@ Finally, these operators appear in the precedence table above but are only meani * `=>` - see [`%{}`](`%{}/1`) * `when` - see [Guards](patterns-and-guards.md#guards) * `<-` - see [`for`](`for/1`) and [`with`](`with/1`) - * `\\` - see [Default arguments](Kernel.html#def/2-default-arguments) + * `\\` - see [Default arguments](`Kernel#def/2-default-arguments`) ## Comparison operators diff --git a/lib/elixir/pages/references/patterns-and-guards.md b/lib/elixir/pages/references/patterns-and-guards.md index f5728f5c6ed..a1fde72414e 100644 --- a/lib/elixir/pages/references/patterns-and-guards.md +++ b/lib/elixir/pages/references/patterns-and-guards.md @@ -274,7 +274,7 @@ Not all expressions are allowed in guard clauses, but only a handful of them. Th ### List of allowed functions and operators -You can find the built-in list of guards [in the `Kernel` module](Kernel.html#guards). Here is an overview: +You can find the built-in list of guards [in the `Kernel` module](`Kernel#guards`). Here is an overview: * comparison operators ([`==`](`==/2`), [`!=`](`!=/2`), [`===`](`===/2`), [`!==`](`!==/2`), [`<`](``](`>/2`), [`>=`](`>=/2`)) diff --git a/lib/elixir/pages/references/typespecs.md b/lib/elixir/pages/references/typespecs.md index 163219d88d3..040d97dcf73 100644 --- a/lib/elixir/pages/references/typespecs.md +++ b/lib/elixir/pages/references/typespecs.md @@ -321,7 +321,7 @@ end ``` This code generates a warning letting you know that you are mistakenly implementing `parse/0` instead of `parse/1`. -You can read more about `@impl` in the [module documentation](Module.html#module-impl). +You can read more about `@impl` in the [module documentation](`Module#module-impl`). ### Using behaviours diff --git a/lib/elixir/pages/references/unicode-syntax.md b/lib/elixir/pages/references/unicode-syntax.md index 40dbfdc6775..8a29f80f010 100644 --- a/lib/elixir/pages/references/unicode-syntax.md +++ b/lib/elixir/pages/references/unicode-syntax.md @@ -81,7 +81,7 @@ Unicode atoms in Elixir follow the identifier rule above with the following modi * `` additionally includes the code point `_` (005F) * `` additionally includes the code point `@` (0040) -Note atoms can also be quoted, which allows any characters, such as `:"hello elixir"`. All Elixir operators are also valid atoms, such as `:+`, `:@`, `:|>`, and others. The full description of valid atoms is available in the ["Atoms" section in the syntax reference](syntax-reference.html#atoms). +Note atoms can also be quoted, which allows any characters, such as `:"hello elixir"`. All Elixir operators are also valid atoms, such as `:+`, `:@`, `:|>`, and others. The full description of valid atoms is available in the ["Atoms" section in the syntax reference](syntax-reference.md#atoms). #### Variables, local calls, and remote calls diff --git a/lib/elixir/scripts/elixir_docs.exs b/lib/elixir/scripts/elixir_docs.exs index 1f7303d91e1..fd20e730c91 100644 --- a/lib/elixir/scripts/elixir_docs.exs +++ b/lib/elixir/scripts/elixir_docs.exs @@ -2,6 +2,7 @@ canonical = System.fetch_env!("CANONICAL") [ + assets: "lib/elixir/pages/images", extras: [ "lib/elixir/pages/getting-started/introduction.md", "lib/elixir/pages/getting-started/basic-types.md", From 740b2d74df617d0fc402616d97330c02761e9b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 29 Jan 2024 10:19:00 +0100 Subject: [PATCH 136/185] Escape rebar3 paths --- lib/mix/lib/mix/tasks/deps.compile.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/mix/lib/mix/tasks/deps.compile.ex b/lib/mix/lib/mix/tasks/deps.compile.ex index 84ba8a3c8b8..5b4f6cdaf13 100644 --- a/lib/mix/lib/mix/tasks/deps.compile.ex +++ b/lib/mix/lib/mix/tasks/deps.compile.ex @@ -207,7 +207,7 @@ defmodule Mix.Tasks.Deps.Compile do {"TERM", "dumb"} ] - cmd = "#{rebar_cmd(dep)} bare compile --paths #{lib_path}" + cmd = "#{escape_path(rebar_cmd(dep))} bare compile --paths #{escape_path(lib_path)}" do_command(dep, config, cmd, false, env) # Check if we have any new symlinks after compilation @@ -220,6 +220,11 @@ defmodule Mix.Tasks.Deps.Compile do true end + defp escape_path(path) do + escape = if match?({:win32, _}, :os.type()), do: "^ ", else: "\\ " + String.replace(path, " ", escape) + end + defp rebar_config(dep) do dep.extra |> Mix.Rebar.dependency_config() From 8344e218a64aad4f362725635ef1cad918c2a722 Mon Sep 17 00:00:00 2001 From: Steve Johns Date: Tue, 30 Jan 2024 13:49:34 +0000 Subject: [PATCH 137/185] docs: fix grammar errors in syntax reference docs (#13302) --- lib/elixir/pages/references/syntax-reference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/elixir/pages/references/syntax-reference.md b/lib/elixir/pages/references/syntax-reference.md index f9ec122012b..2feff8f84d7 100644 --- a/lib/elixir/pages/references/syntax-reference.md +++ b/lib/elixir/pages/references/syntax-reference.md @@ -92,7 +92,7 @@ Variables in Elixir must start with an underscore or a Unicode letter that is no ### Non-qualified calls (local calls) -Non-qualified calls, such as `add(1, 2)`, must start with characters and then follow the same rules as as variables, which are optionally followed by parentheses, and then arguments. +Non-qualified calls, such as `add(1, 2)`, must start with characters and then follow the same rules as variables, which are optionally followed by parentheses, and then arguments. Parentheses are required for zero-arity calls (i.e. calls without arguments), to avoid ambiguity with variables. If parentheses are used, they must immediately follow the function name *without spaces*. For example, `add (1, 2)` is a syntax error, since `(1, 2)` is treated as an invalid block which is attempted to be given as a single argument to `add`. @@ -104,7 +104,7 @@ As many programming languages, Elixir also support operators as non-qualified ca ### Qualified calls (remote calls) -Qualified calls, such as `Math.add(1, 2)`, must start with characters and then follow the same rules as as variables, which are optionally followed by parentheses, and then arguments. Qualified calls also support operators, such as `Kernel.+(1, 2)`. Elixir also allows the function name to be written between double- or single-quotes, allowing any character in between the quotes, such as `Math."++add++"(1, 2)`. +Qualified calls, such as `Math.add(1, 2)`, must start with characters and then follow the same rules as variables, which are optionally followed by parentheses, and then arguments. Qualified calls also support operators, such as `Kernel.+(1, 2)`. Elixir also allows the function name to be written between double- or single-quotes, allowing any character in between the quotes, such as `Math."++add++"(1, 2)`. Similar to non-qualified calls, parentheses have different meaning for zero-arity calls (i.e. calls without arguments). If parentheses are used, such as `mod.fun()`, it means a function call. If parenthesis are skipped, such as `map.field`, it means accessing a field of a map. From 60dcd143900a4f39fc25b6fd8877ca28ad508fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 30 Jan 2024 20:21:23 +0100 Subject: [PATCH 138/185] Fix Rebar3 env var with spaces (#13303) --- lib/mix/lib/mix/rebar.ex | 4 ++-- lib/mix/lib/mix/tasks/deps.compile.ex | 2 +- lib/mix/test/mix/rebar_test.exs | 23 ++++++++++++++++++++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/mix/lib/mix/rebar.ex b/lib/mix/lib/mix/rebar.ex index 0704fa4bf37..2a38dd36ee6 100644 --- a/lib/mix/lib/mix/rebar.ex +++ b/lib/mix/lib/mix/rebar.ex @@ -36,7 +36,7 @@ defmodule Mix.Rebar do @doc """ Returns the path to the available `rebar` command. """ - # TODO: Remove on Elixir v1.18 because phx_new and other installers rely on it. + # TODO: Remove on Elixir v1.20 because phx_new and other installers rely on it. def rebar_cmd(:rebar) do Mix.shell().error("[warning] :rebar is no longer supported in Mix, falling back to :rebar3") rebar_cmd(:rebar3) @@ -218,7 +218,7 @@ defmodule Mix.Rebar do defp wrap_cmd(rebar) do cond do not match?({:win32, _}, :os.type()) -> - rebar + String.replace(rebar, " ", "\\ ") String.ends_with?(rebar, ".cmd") -> "\"#{String.replace(rebar, "/", "\\")}\"" diff --git a/lib/mix/lib/mix/tasks/deps.compile.ex b/lib/mix/lib/mix/tasks/deps.compile.ex index 5b4f6cdaf13..acc51a6c25e 100644 --- a/lib/mix/lib/mix/tasks/deps.compile.ex +++ b/lib/mix/lib/mix/tasks/deps.compile.ex @@ -207,7 +207,7 @@ defmodule Mix.Tasks.Deps.Compile do {"TERM", "dumb"} ] - cmd = "#{escape_path(rebar_cmd(dep))} bare compile --paths #{escape_path(lib_path)}" + cmd = "#{rebar_cmd(dep)} bare compile --paths #{escape_path(lib_path)}" do_command(dep, config, cmd, false, env) # Check if we have any new symlinks after compilation diff --git a/lib/mix/test/mix/rebar_test.exs b/lib/mix/test/mix/rebar_test.exs index a5154b87a83..c187e2d6048 100644 --- a/lib/mix/test/mix/rebar_test.exs +++ b/lib/mix/test/mix/rebar_test.exs @@ -219,7 +219,7 @@ defmodule Mix.RebarTest do # We run only on Unix because Windows has a hard time # removing the Rebar executable after executed. @tag :unix - test "applies variables from :system_env option when compiling dependencies" do + test "applies variables from :system_env option on config/compilation" do in_tmp("applies variables from system_env", fn -> Mix.Project.push(RebarAsDepWithEnv) @@ -233,6 +233,27 @@ defmodule Mix.RebarTest do end) end + # We run only on Unix because Windows has a hard time + # removing the Rebar executable after executed. + @tag :unix + test "gets and compiles dependencies with MIX_REBAR3 with spaces" do + in_tmp("rebar3 env with spaces", fn -> + File.cp!(Mix.Rebar.local_rebar_path(:rebar3), "rebar3") + System.put_env("MIX_REBAR3", Path.absname("rebar3")) + assert Mix.Rebar.rebar_cmd(:rebar3) =~ " " + + Mix.Project.push(RebarAsDep) + Mix.Tasks.Deps.Get.run([]) + assert_received {:mix_shell, :info, ["* Getting git_rebar " <> _]} + + Mix.Tasks.Deps.Compile.run([]) + assert_received {:mix_shell, :run, ["===> Compiling git_rebar\n"]} + assert_received {:mix_shell, :run, ["===> Compiling rebar_dep\n"]} + end) + after + System.delete_env("MIX_REBAR3") + end + test "gets and compiles dependencies with Mix" do in_tmp("get and compile dependencies with Mix", fn -> Mix.Project.push(RebarAsDep) From cf8c28c34bdeb3b8494732277d288dd4fdef089c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 31 Jan 2024 10:12:50 +0100 Subject: [PATCH 139/185] Fix autocompletion on Erlang/OTP 26, closes #13307 --- lib/iex/lib/iex/autocomplete.ex | 4 +--- lib/iex/test/iex/autocomplete_test.exs | 7 ++----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/iex/lib/iex/autocomplete.ex b/lib/iex/lib/iex/autocomplete.ex index 3ea80624160..425d0f89f03 100644 --- a/lib/iex/lib/iex/autocomplete.ex +++ b/lib/iex/lib/iex/autocomplete.ex @@ -197,9 +197,7 @@ defmodule IEx.Autocomplete do end defp expand_signatures([_ | _] = signatures, _shell) do - [head | tail] = Enum.sort(signatures, &(String.length(&1) <= String.length(&2))) - if tail != [], do: IO.write("\n" <> (tail |> Enum.reverse() |> Enum.join("\n"))) - yes("", [head]) + yes("", Enum.sort_by(signatures, &String.length/1)) end defp expand_signatures([], shell), do: expand_local_or_var("", shell) diff --git a/lib/iex/test/iex/autocomplete_test.exs b/lib/iex/test/iex/autocomplete_test.exs index b3ef186bde6..3f7d6c4eb39 100644 --- a/lib/iex/test/iex/autocomplete_test.exs +++ b/lib/iex/test/iex/autocomplete_test.exs @@ -492,11 +492,8 @@ defmodule IEx.AutocompleteTest do eval("import Enum; import Protocol") - assert ExUnit.CaptureIO.capture_io(fn -> - send(self(), expand(~c"reduce(")) - end) == "\nreduce(enumerable, acc, fun)" - - assert_received {:yes, ~c"", [~c"reduce(enumerable, fun)"]} + assert expand(~c"reduce(") == + {:yes, ~c"", [~c"reduce(enumerable, fun)", ~c"reduce(enumerable, acc, fun)"]} assert expand(~c"take(") == {:yes, ~c"", [~c"take(enumerable, amount)"]} assert expand(~c"derive(") == {:yes, ~c"", [~c"derive(protocol, module, options \\\\ [])"]} From e0658bb55a744e33a70c982c992666addc4feb38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 31 Jan 2024 10:24:07 +0100 Subject: [PATCH 140/185] Release v1.16.1 --- CHANGELOG.md | 22 ++++++++++++++++++++++ VERSION | 2 +- bin/elixir | 2 +- bin/elixir.bat | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29fd5111f37..0df54f9b90b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,28 @@ Another [ExDoc](https://github.com/elixir-lang/ex_doc) feature we have incorpora Finally, we have started enriching our documentation with [Mermaid.js](https://mermaid.js.org/) diagrams. You can find examples in the [GenServer](https://hexdocs.pm/elixir/main/GenServer.html) and [Supervisor](https://hexdocs.pm/elixir/main/Supervisor.html) docs. +## v1.16.1 (2024-01-31) + +### 1. Bug fixes + +#### Elixir + + * [Code] Fix `Code.quoted_to_algebra/2` for operator with :do key as operand + * [Kernel.ParallelCompiler] Do not crash parallel compiler when it receives diagnostics from additional code evaluation + * [Kernel.ParallelCompiler] Always log errors at the end of compilation + * [String] Fix `String.capitalize/1` with a single codepoint + +#### IEx + + * [IEx] Fix autocompletion of function signatures on Erlang/OTP 26 + * [IEx] Do not assume `$HOME` is set + +#### Mix + + * [mix deps.compile] Handle compilation of rebar3 dependencies when rebar3 is on a path with spaces on Unix + * [mix test] Properly resolve relative paths when running tests from individual files + * [mix test] Properly resolve Windows paths when running tests from individual files + ## v1.16.0 (2023-12-22) ### 1. Enhancements diff --git a/VERSION b/VERSION index 71bd5d9ee64..bbb75a1b664 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.0 \ No newline at end of file +1.16.1 \ No newline at end of file diff --git a/bin/elixir b/bin/elixir index 7d660e025e9..783c566d290 100755 --- a/bin/elixir +++ b/bin/elixir @@ -1,7 +1,7 @@ #!/bin/sh set -e -ELIXIR_VERSION=1.16.0 +ELIXIR_VERSION=1.16.1 if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then cat <&2 diff --git a/bin/elixir.bat b/bin/elixir.bat index 604dec081b0..fb53b50c456 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -1,6 +1,6 @@ @if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) -set ELIXIR_VERSION=1.16.0 +set ELIXIR_VERSION=1.16.1 setlocal enabledelayedexpansion if ""%1""=="""" if ""%2""=="""" goto documentation From 7098ac8a169231edd415d6f4a9c5fa874efaa7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 1 Feb 2024 12:38:11 +0100 Subject: [PATCH 141/185] Preserve . semantics in Path.relative_to, closes #13310 --- lib/elixir/lib/path.ex | 2 ++ lib/elixir/test/elixir/path_test.exs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/elixir/lib/path.ex b/lib/elixir/lib/path.ex index bba4de5ad70..70817658e53 100644 --- a/lib/elixir/lib/path.ex +++ b/lib/elixir/lib/path.ex @@ -424,6 +424,8 @@ defmodule Path do defp relative_to_unforced(_, _, original), do: join(original) defp relative_to_forced(path, path, _original), do: "." + defp relative_to_forced(["."], _path, _original), do: "." + defp relative_to_forced(path, ["."], _original), do: join(path) defp relative_to_forced([h | t1], [h | t2], original), do: relative_to_forced(t1, t2, original) # this should only happen if we have two paths on different drives on windows diff --git a/lib/elixir/test/elixir/path_test.exs b/lib/elixir/test/elixir/path_test.exs index 99c43a48bc6..dd775c707be 100644 --- a/lib/elixir/test/elixir/path_test.exs +++ b/lib/elixir/test/elixir/path_test.exs @@ -316,6 +316,8 @@ defmodule PathTest do assert Path.relative_to("./foo/../bar/..", File.cwd!()) == "." # both relative + assert Path.relative_to("usr/local/foo", ".") == "usr/local/foo" + assert Path.relative_to(".", "usr/local/foo") == "." assert Path.relative_to("usr/local/foo", "usr/local") == "foo" assert Path.relative_to("usr/local/foo", "etc") == "../usr/local/foo" assert Path.relative_to(~c"usr/local/foo", "etc") == "../usr/local/foo" From d8cc841ab06493b806b14c95646be5b703a29de8 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Mon, 5 Feb 2024 03:40:16 -0500 Subject: [PATCH 142/185] Include from_brackets metadata in all cases (#13317) --- lib/elixir/src/elixir_parser.yrl | 6 ++-- .../test/elixir/kernel/tracers_test.exs | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index ce6afd32172..f2495e93b34 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -292,13 +292,13 @@ bracket_arg -> open_bracket container_expr close_bracket : build_access_arg('$1' bracket_arg -> open_bracket container_expr ',' close_bracket : build_access_arg('$1', '$2', '$4'). bracket_arg -> open_bracket container_expr ',' container_args close_bracket : error_too_many_access_syntax('$3'). -bracket_expr -> dot_bracket_identifier bracket_arg : build_access(build_no_parens('$1', nil), '$2'). +bracket_expr -> dot_bracket_identifier bracket_arg : build_access(build_no_parens('$1', nil), meta_with_from_brackets('$2')). bracket_expr -> access_expr bracket_arg : build_access('$1', meta_with_from_brackets('$2')). bracket_at_expr -> at_op_eol dot_bracket_identifier bracket_arg : - build_access(build_unary_op('$1', build_no_parens('$2', nil)), '$3'). + build_access(build_unary_op('$1', build_no_parens('$2', nil)), meta_with_from_brackets('$3')). bracket_at_expr -> at_op_eol access_expr bracket_arg : - build_access(build_unary_op('$1', '$2'), '$3'). + build_access(build_unary_op('$1', '$2'), meta_with_from_brackets('$3')). %% Blocks diff --git a/lib/elixir/test/elixir/kernel/tracers_test.exs b/lib/elixir/test/elixir/kernel/tracers_test.exs index 08826bf324a..7374ea41efd 100644 --- a/lib/elixir/test/elixir/kernel/tracers_test.exs +++ b/lib/elixir/test/elixir/kernel/tracers_test.exs @@ -217,6 +217,35 @@ defmodule Kernel.TracersTest do assert meta[:from_interpolation] end + test "traces bracket access" do + compile_string(""" + foo = %{bar: 3} + foo[:bar] + """) + + assert_receive {{:remote_function, meta, Access, :get, 2}, _env} + assert meta[:from_brackets] + + compile_string(""" + defmodule Foo do + @foo %{bar: 3} + def a() do + @foo[:bar] + end + end + """) + + assert_receive {{:remote_function, meta, Access, :get, 2}, _env} + assert meta[:from_brackets] + + compile_string(""" + %{bar: 3}[:bar] + """) + + assert_receive {{:remote_function, meta, Access, :get, 2}, _env} + assert meta[:from_brackets] + end + """ # Make sure this module is compiled with column information defmodule MacroWithColumn do From ef3e2796ef569af74cb048d10bd12b82bd43acae Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Fri, 9 Feb 2024 16:49:27 +0900 Subject: [PATCH 143/185] Update argument error message when matching with <> (#13325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update argument error message when matching with <> * Update lib/elixir/lib/kernel.ex Co-authored-by: José Valim --------- Co-authored-by: José Valim --- lib/elixir/lib/kernel.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/kernel.ex b/lib/elixir/lib/kernel.ex index 146d3ab5b14..30709554afd 100644 --- a/lib/elixir/lib/kernel.ex +++ b/lib/elixir/lib/kernel.ex @@ -2076,8 +2076,9 @@ defmodule Kernel do defp invalid_concat_left_argument_error(arg) do :erlang.error( ArgumentError.exception( - "the left argument of <> operator inside a match should always be a literal " <> - "binary because its size can't be verified. Got: #{arg}" + "cannot perform prefix match because the left operand of <> has unknown size. " <> + "The left operand of <> inside a match should either be a literal binary or " <> + "an existing variable with the pin operator (such as ^some_var). Got: #{arg}" ) ) end From bf915aa2e79eee723764d3805b16f0bf824fcf67 Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Fri, 9 Feb 2024 17:02:56 +0900 Subject: [PATCH 144/185] Fix <> match test (#13327) --- lib/elixir/test/elixir/kernel/binary_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/test/elixir/kernel/binary_test.exs b/lib/elixir/test/elixir/kernel/binary_test.exs index fd26ad3aee3..dadcfaee67a 100644 --- a/lib/elixir/test/elixir/kernel/binary_test.exs +++ b/lib/elixir/test/elixir/kernel/binary_test.exs @@ -119,7 +119,7 @@ defmodule Kernel.BinaryTest do Code.eval_string(~s["foo" <> 1]) end - message = ~r"left argument of <> operator inside a match" + message = ~r"cannot perform prefix match because the left operand of <> has unknown size." assert_raise ArgumentError, message, fn -> Code.eval_string(~s[a <> "b" = "ab"]) From f3e669c471fd3aef2bccee85f1a4cb74042e58ab Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Tue, 13 Feb 2024 07:38:49 +0900 Subject: [PATCH 145/185] Fix :trim_doc example in doc (#13340) --- lib/elixir/lib/file.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/lib/file.ex b/lib/elixir/lib/file.ex index ecf4b8550dd..df1664dd658 100644 --- a/lib/elixir/lib/file.ex +++ b/lib/elixir/lib/file.ex @@ -1764,7 +1764,7 @@ defmodule File do ## Examples # Read a utf8 text file which may include BOM - File.stream!("./test/test.txt", encoding: :utf8, trim_bom: true) + File.stream!("./test/test.txt", [:trim_bom, encoding: :utf8]) # Read in 2048 byte chunks rather than lines File.stream!("./test/test.data", 2048) From e5ec5ce7f6ab87863295c57da6e780076e64f63d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 15 Feb 2024 12:21:23 +0100 Subject: [PATCH 146/185] Add API for deleting SCM This is used by Hex when its application is stopped to revert its changes to Mix state. --- lib/mix/lib/mix/scm.ex | 8 ++++++++ lib/mix/test/mix/scm_test.exs | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/lib/mix/lib/mix/scm.ex b/lib/mix/lib/mix/scm.ex index 5efeb0c67d8..92170f8ebad 100644 --- a/lib/mix/lib/mix/scm.ex +++ b/lib/mix/lib/mix/scm.ex @@ -131,6 +131,14 @@ defmodule Mix.SCM do Mix.State.get(:scm) end + @doc """ + Deletes the given SCM from the list of available SCMs. + """ + @doc since: "1.16.2" + def delete(mod) when is_atom(mod) do + Mix.State.update(:scm, &List.delete(&1, mod)) + end + @doc """ Prepends the given SCM module to the list of available SCMs. """ diff --git a/lib/mix/test/mix/scm_test.exs b/lib/mix/test/mix/scm_test.exs index 4c107d5b5a8..2ec3e515f50 100644 --- a/lib/mix/test/mix/scm_test.exs +++ b/lib/mix/test/mix/scm_test.exs @@ -12,10 +12,14 @@ defmodule Mix.SCMTest do test "prepends an SCM" do Mix.SCM.prepend(Hello) assert Enum.at(Mix.SCM.available(), 0) == Hello + Mix.SCM.delete(Hello) + assert Hello not in Mix.SCM.available() end test "appends an SCM" do Mix.SCM.append(Hello) assert Enum.at(Mix.SCM.available(), -1) == Hello + Mix.SCM.delete(Hello) + assert Hello not in Mix.SCM.available() end end From ff309747b7682d56895a2cf1e13e4f3ed669d5d7 Mon Sep 17 00:00:00 2001 From: Philip Munksgaard Date: Sat, 17 Feb 2024 12:51:07 +0100 Subject: [PATCH 147/185] Escape pinned values when computing diff (#13354) Take these two tests: ```elixir test "correctly colored" do assert [{:foo}] = [{:bar}] end test "incorrectly colored" do val = [{:foo}] assert ^val = [{:bar}] end ``` That's because when diffing a pin, we were not converting the underlying diff context from match to ===. This fixes #13348 --- lib/ex_unit/lib/ex_unit/diff.ex | 4 ++-- lib/ex_unit/test/ex_unit/diff_test.exs | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/diff.ex b/lib/ex_unit/lib/ex_unit/diff.ex index 051fd250ccc..e16b7656f80 100644 --- a/lib/ex_unit/lib/ex_unit/diff.ex +++ b/lib/ex_unit/lib/ex_unit/diff.ex @@ -286,10 +286,10 @@ defmodule ExUnit.Diff do defp diff_pin({:^, _, [var]} = pin, right, %{pins: pins} = env) do identifier = var_context(var) %{^identifier => pin_value} = pins - {diff, post_env} = diff_value(pin_value, right, env) + {diff, post_env} = diff_value(pin_value, right, %{env | context: :===}) diff_left = update_diff_meta(pin, not diff.equivalent?) - {%{diff | left: diff_left}, post_env} + {%{diff | left: diff_left}, %{post_env | context: :match}} end # Vars diff --git a/lib/ex_unit/test/ex_unit/diff_test.exs b/lib/ex_unit/test/ex_unit/diff_test.exs index e443c42a08b..04340b8db5a 100644 --- a/lib/ex_unit/test/ex_unit/diff_test.exs +++ b/lib/ex_unit/test/ex_unit/diff_test.exs @@ -251,7 +251,12 @@ defmodule ExUnit.DiffTest do refute_diff([:a, [:c, :b]] = [:a, [:b, :c]], "[:a, [-:c-, :b]]", "[:a, [:b, +:c+]]") refute_diff(:a = [:a, [:b, :c]], "-:a-", "+[:a, [:b, :c]]+") - pins = %{{:a, nil} => :a, {:b, nil} => :b, {:list_ab, nil} => [:a, :b]} + pins = %{ + {:a, nil} => :a, + {:b, nil} => :b, + {:list_ab, nil} => [:a, :b], + {:list_tuple, nil} => [{:foo}] + } assert_diff(x = [], [x: []], pins) assert_diff(x = [:a, :b], [x: [:a, :b]], pins) @@ -282,6 +287,9 @@ defmodule ExUnit.DiffTest do refute_diff([:a, :b] = :a, "-[:a, :b]-", "+:a+") refute_diff([:foo] = [:foo, {:a, :b, :c}], "[:foo]", "[:foo, +{:a, :b, :c}+]") + + refute_diff([{:foo}] = [{:bar}], "[{-:foo-}]", "[{+:bar+}]") + refute_diff(^list_tuple = [{:bar}], "-^list_tuple-", "[{+:bar+}]", pins) end test "improper lists" do From 677ad618e9fd78c2c73497b67f9768cb87517cce Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Sat, 24 Feb 2024 00:02:00 +0900 Subject: [PATCH 148/185] Fix charlist formatting issue on '\"' (#13364) --- lib/elixir/lib/code/formatter.ex | 34 +++++++++++-------- .../elixir/code_formatter/literals_test.exs | 11 ++++-- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex index 5c84f675feb..6d50afb7e15 100644 --- a/lib/elixir/lib/code/formatter.ex +++ b/lib/elixir/lib/code/formatter.ex @@ -6,7 +6,8 @@ defmodule Code.Formatter do @double_heredoc "\"\"\"" @single_quote "'" @single_heredoc "'''" - @sigil_c "~c\"" + @sigil_c_double "~c\"" + @sigil_c_single "~c'" @sigil_c_heredoc "~c\"\"\"" @newlines 2 @min_line 0 @@ -299,7 +300,7 @@ defmodule Code.Formatter do remote_to_algebra(quoted, context, state) meta[:delimiter] == ~s['''] -> - {opener, quotes} = get_charlist_quotes(true, state) + {opener, quotes} = get_charlist_quotes(:heredoc, state) {doc, state} = entries @@ -309,7 +310,7 @@ defmodule Code.Formatter do {force_unfit(doc), state} true -> - {opener, quotes} = get_charlist_quotes(false, state) + {opener, quotes} = get_charlist_quotes({:regular, entries}, state) list_interpolation_to_algebra(entries, quotes, state, opener, quotes) end end @@ -368,13 +369,14 @@ defmodule Code.Formatter do defp quoted_to_algebra({:__block__, meta, [list]}, _context, state) when is_list(list) do case meta[:delimiter] do ~s['''] -> - {opener, quotes} = get_charlist_quotes(true, state) + {opener, quotes} = get_charlist_quotes(:heredoc, state) string = list |> List.to_string() |> escape_heredoc(quotes) {opener |> concat(string) |> concat(quotes) |> force_unfit(), state} ~s['] -> - {opener, quotes} = get_charlist_quotes(false, state) - string = list |> List.to_string() |> escape_string(quotes) + string = list |> List.to_string() + {opener, quotes} = get_charlist_quotes({:regular, [string]}, state) + string = escape_string(string, quotes) {opener |> concat(string) |> concat(quotes), state} _other -> @@ -2410,19 +2412,23 @@ defmodule Code.Formatter do {left, right} end - defp get_charlist_quotes(_heredoc = false, state) do + defp get_charlist_quotes(:heredoc, state) do if state.normalize_charlists_as_sigils do - {@sigil_c, @double_quote} + {@sigil_c_heredoc, @double_heredoc} else - {@single_quote, @single_quote} + {@single_heredoc, @single_heredoc} end end - defp get_charlist_quotes(_heredoc = true, state) do - if state.normalize_charlists_as_sigils do - {@sigil_c_heredoc, @double_heredoc} - else - {@single_heredoc, @single_heredoc} + defp get_charlist_quotes({:regular, chunks}, state) do + cond do + !state.normalize_charlists_as_sigils -> {@single_quote, @single_quote} + Enum.any?(chunks, &has_double_quote?/1) -> {@sigil_c_single, @single_quote} + true -> {@sigil_c_double, @double_quote} end end + + defp has_double_quote?(chunk) do + is_binary(chunk) and chunk =~ @double_quote + end end diff --git a/lib/elixir/test/elixir/code_formatter/literals_test.exs b/lib/elixir/test/elixir/code_formatter/literals_test.exs index ec614b6949e..7f5a7632faf 100644 --- a/lib/elixir/test/elixir/code_formatter/literals_test.exs +++ b/lib/elixir/test/elixir/code_formatter/literals_test.exs @@ -210,10 +210,15 @@ defmodule Code.Formatter.LiteralsTest do test "with escapes" do assert_format ~S['f\a\b\ro'], ~S[~c"f\a\b\ro"] assert_format ~S['single \' quote'], ~S[~c"single ' quote"] - assert_format ~S['double " quote'], ~S[~c"double \" quote"] + assert_format ~S['double " quote'], ~S[~c'double " quote'] + assert_format ~S['escaped \" quote'], ~S[~c'escaped \" quote'] + assert_format ~S['\\"'], ~S[~c'\\"'] assert_same ~S['f\a\b\ro'], @keep_charlists assert_same ~S['single \' quote'], @keep_charlists + assert_same ~S['double " quote'], @keep_charlists + assert_same ~S['escaped \" quote'], @keep_charlists + assert_same ~S['\\"'], @keep_charlists end test "keeps literal new lines" do @@ -235,13 +240,15 @@ defmodule Code.Formatter.LiteralsTest do test "with interpolation" do assert_format ~S['one #{2} three'], ~S[~c"one #{2} three"] + assert_format ~S['#{1}\n \\ " \"'], ~S[~c'#{1}\n \\ " \"'] assert_same ~S['one #{2} three'], @keep_charlists + assert_same ~S['#{1}\n \\ " \"'], @keep_charlists end test "with escape and interpolation" do assert_format ~S['one\n\'#{2}\'\nthree'], ~S[~c"one\n'#{2}'\nthree"] - assert_format ~S['one\n"#{2}"\nthree'], ~S[~c"one\n\"#{2}\"\nthree"] + assert_format ~S['one\n"#{2}"\nthree'], ~S[~c'one\n"#{2}"\nthree'] assert_same ~S['one\n\'#{2}\'\nthree'], @keep_charlists end From a99a9cf47ae0bb49e8e2a44299ccef09c8472fac Mon Sep 17 00:00:00 2001 From: sabiwara Date: Sat, 24 Feb 2024 00:11:43 +0900 Subject: [PATCH 149/185] Fix test due to backport --- lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index f6deeffb61f..27b6723ba1e 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -708,10 +708,10 @@ defmodule Code.Normalizer.QuotedASTTest do ~S[~c"one\n'#{2}'\nthree"] assert Code.string_to_quoted!(~S/'one\n"#{2}"\nthree'/) |> quoted_to_string(escape: false) == - ~s[~c"one\n\\"\#{2}\\"\nthree"] + ~s[~c'one\n"\#{2}"\nthree'] assert Code.string_to_quoted!(~S/'one\n"#{2}"\nthree'/) |> quoted_to_string() == - ~S[~c"one\n\"#{2}\"\nthree"] + ~S[~c'one\n"#{2}"\nthree'] end test "atoms" do From 2512cbaff064652872ceca124e45779bbf732d76 Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Sat, 24 Feb 2024 22:20:24 +0900 Subject: [PATCH 150/185] Fix doctest example since Inspect.MapSet changed (#13366) --- lib/ex_unit/lib/ex_unit/doc_test.ex | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/doc_test.ex b/lib/ex_unit/lib/ex_unit/doc_test.ex index 7d99db93399..2119cf123e0 100644 --- a/lib/ex_unit/lib/ex_unit/doc_test.ex +++ b/lib/ex_unit/lib/ex_unit/doc_test.ex @@ -96,9 +96,9 @@ defmodule ExUnit.DocTest do values are treated as comments in Elixir code due to the leading `#` sign, they require special care when being used in doctests. - Imagine you have a map that contains a MapSet and is printed as: + Imagine you have a map that contains a `DateTime` and is printed as: - %{users: #MapSet<[:foo, :bar]>} + %{datetime: #DateTime<2023-06-26 09:30:00+09:00 JST Asia/Tokyo>} If you try to match on such an expression, `doctest` will fail to compile. There are two ways to resolve this. @@ -106,20 +106,20 @@ defmodule ExUnit.DocTest do The first is to rely on the fact that doctest can compare internal structures as long as they are at the root. So one could write: - iex> map = %{users: Enum.into([:foo, :bar], MapSet.new())} - iex> map.users - #MapSet<[:foo, :bar]> + iex> map = %{datetime: DateTime.from_naive!(~N[2023-06-26T09:30:00], "Asia/Tokyo")} + iex> map.datetime + #DateTime<2023-06-26 09:30:00+09:00 JST Asia/Tokyo> Whenever a doctest starts with "#Name<", `doctest` will perform a string comparison. For example, the above test will perform the following match: - inspect(map.users) == "#MapSet<[:foo, :bar]>" + inspect(map.datetime) == "#DateTime<2023-06-26 09:30:00+09:00 JST Asia/Tokyo>" Alternatively, since doctest results are actually evaluated, you can have - the MapSet building expression as the doctest result: + the `DateTime` building expression as the doctest result: - iex> %{users: Enum.into([:foo, :bar], MapSet.new())} - %{users: Enum.into([:foo, :bar], MapSet.new())} + iex> %{datetime: DateTime.from_naive!(~N[2023-06-26T09:30:00], "Asia/Tokyo")} + %{datetime: DateTime.from_naive!(~N[2023-06-26T09:30:00], "Asia/Tokyo")} The downside of this approach is that the doctest result is not really what users would see in the terminal. From dd004539c692130c09ec169287867e5ddc349e08 Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Fri, 16 Feb 2024 16:47:28 +0900 Subject: [PATCH 151/185] Fixes to support 0TP27 (#13351) * Fix non-deterministic key-value tests * Fix non-deterministic Enum tests * Fix :only option when deriving Inspect * Float.ceil and Float.floor return -0.0 for negative numbers * Fix non-deterministic Registry doctest * Simplify check --- lib/elixir/lib/enum.ex | 5 +---- lib/elixir/lib/float.ex | 16 +++++++++++++++- lib/elixir/lib/inspect.ex | 5 +++-- lib/elixir/lib/map.ex | 4 ++-- lib/elixir/lib/registry.ex | 8 ++++---- lib/elixir/test/elixir/enum_test.exs | 8 ++++---- lib/elixir/test/elixir/float_test.exs | 22 +++++++++++++++++++--- 7 files changed, 48 insertions(+), 20 deletions(-) diff --git a/lib/elixir/lib/enum.ex b/lib/elixir/lib/enum.ex index 373da0a6636..7b7127d60dd 100644 --- a/lib/elixir/lib/enum.ex +++ b/lib/elixir/lib/enum.ex @@ -3793,9 +3793,6 @@ defmodule Enum do iex> Enum.unzip([{:a, 1}, {:b, 2}, {:c, 3}]) {[:a, :b, :c], [1, 2, 3]} - iex> Enum.unzip(%{a: 1, b: 2}) - {[:a, :b], [1, 2]} - """ @spec unzip(t) :: {[element], [element]} @@ -4051,7 +4048,7 @@ defmodule Enum do ...> end) [{1, 2, 3}, {1, 2, 3}] - iex> enums = [[1, 2], %{a: 3, b: 4}, [5, 6]] + iex> enums = [[1, 2], [a: 3, b: 4], [5, 6]] ...> Enum.zip_reduce(enums, [], fn elements, acc -> ...> [List.to_tuple(elements) | acc] ...> end) diff --git a/lib/elixir/lib/float.ex b/lib/elixir/lib/float.ex index 62b5dad25f8..63fd3a55548 100644 --- a/lib/elixir/lib/float.ex +++ b/lib/elixir/lib/float.ex @@ -275,6 +275,8 @@ defmodule Float do -56.0 iex> Float.ceil(34.251, 2) 34.26 + iex> Float.ceil(-0.01) + -0.0 """ @spec ceil(float, precision_range) :: float @@ -332,6 +334,8 @@ defmodule Float do -6.0 iex> Float.round(12.341444444444441, 15) 12.341444444444441 + iex> Float.round(-0.01) + -0.0 """ @spec round(float, precision_range) :: float @@ -340,8 +344,13 @@ defmodule Float do # and could be implemented in the future. def round(float, precision \\ 0) + def round(float, 0) when float == 0.0, do: float + def round(float, 0) when is_float(float) do - float |> :erlang.round() |> :erlang.float() + case float |> :erlang.round() |> :erlang.float() do + zero when zero == 0.0 and float < 0.0 -> -0.0 + rounded -> rounded + end end def round(float, precision) when is_float(float) and precision in @precision_range do @@ -365,6 +374,8 @@ defmodule Float do case rounding do :ceil when sign === 0 -> 1 / power_of_10(precision) :floor when sign === 1 -> -1 / power_of_10(precision) + :ceil when sign === 1 -> -0.0 + :half_up when sign === 1 -> -0.0 _ -> 0.0 end @@ -394,6 +405,9 @@ defmodule Float do boundary = den <<< 52 cond do + num == 0 and sign == 1 -> + -0.0 + num == 0 -> 0.0 diff --git a/lib/elixir/lib/inspect.ex b/lib/elixir/lib/inspect.ex index d5753b8d844..92dc6765564 100644 --- a/lib/elixir/lib/inspect.ex +++ b/lib/elixir/lib/inspect.ex @@ -535,7 +535,8 @@ end defimpl Inspect, for: Any do defmacro __deriving__(module, struct, options) do - fields = Map.keys(struct) -- [:__exception__, :__struct__] + fields = Enum.sort(Map.keys(struct) -- [:__exception__, :__struct__]) + only = Keyword.get(options, :only, fields) except = Keyword.get(options, :except, []) optional = Keyword.get(options, :optional, []) @@ -545,7 +546,7 @@ defimpl Inspect, for: Any do :ok = validate_option(:optional, optional, fields, module) inspect_module = - if fields == only and except == [] do + if fields == Enum.sort(only) and except == [] do Inspect.Map else Inspect.Any diff --git a/lib/elixir/lib/map.ex b/lib/elixir/lib/map.ex index 8082b401686..a9507dc60bc 100644 --- a/lib/elixir/lib/map.ex +++ b/lib/elixir/lib/map.ex @@ -147,7 +147,7 @@ defmodule Map do ## Examples - iex> Map.keys(%{a: 1, b: 2}) + Map.keys(%{a: 1, b: 2}) [:a, :b] """ @@ -161,7 +161,7 @@ defmodule Map do ## Examples - iex> Map.values(%{a: 1, b: 2}) + Map.values(%{a: 1, b: 2}) [1, 2] """ diff --git a/lib/elixir/lib/registry.ex b/lib/elixir/lib/registry.ex index 4177e35bae7..4ee6661caca 100644 --- a/lib/elixir/lib/registry.ex +++ b/lib/elixir/lib/registry.ex @@ -1301,16 +1301,16 @@ defmodule Registry do iex> Registry.start_link(keys: :unique, name: Registry.SelectAllTest) iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "hello", :value) iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "world", :value) - iex> Registry.select(Registry.SelectAllTest, [{{:"$1", :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]) - [{"world", self(), :value}, {"hello", self(), :value}] + iex> Registry.select(Registry.SelectAllTest, [{{:"$1", :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]) |> Enum.sort() + [{"hello", self(), :value}, {"world", self(), :value}] Get all keys in the registry: iex> Registry.start_link(keys: :unique, name: Registry.SelectAllTest) iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "hello", :value) iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "world", :value) - iex> Registry.select(Registry.SelectAllTest, [{{:"$1", :_, :_}, [], [:"$1"]}]) - ["world", "hello"] + iex> Registry.select(Registry.SelectAllTest, [{{:"$1", :_, :_}, [], [:"$1"]}]) |> Enum.sort() + ["hello", "world"] """ @doc since: "1.9.0" diff --git a/lib/elixir/test/elixir/enum_test.exs b/lib/elixir/test/elixir/enum_test.exs index ffb9c0bf8f0..96abd36eb69 100644 --- a/lib/elixir/test/elixir/enum_test.exs +++ b/lib/elixir/test/elixir/enum_test.exs @@ -56,9 +56,9 @@ defmodule EnumTest do end test "mix and match" do - enums = [[1, 2], %{a: 3, b: 4}, [5, 6]] + enums = [[1, 2], 3..4, [5, 6]] result = Enum.zip_reduce(enums, [], fn elements, acc -> [List.to_tuple(elements) | acc] end) - assert result == [{2, {:b, 4}, 6}, {1, {:a, 3}, 5}] + assert result == [{2, 4, 6}, {1, 3, 5}] end end @@ -412,7 +412,7 @@ defmodule EnumTest do assert Enum.into([a: 1, b: 2], %{c: 3}) == %{a: 1, b: 2, c: 3} assert Enum.into(MapSet.new(a: 1, b: 2), %{}) == %{a: 1, b: 2} assert Enum.into(MapSet.new(a: 1, b: 2), %{c: 3}) == %{a: 1, b: 2, c: 3} - assert Enum.into(%{a: 1, b: 2}, []) == [a: 1, b: 2] + assert Enum.into(%{a: 1, b: 2}, []) |> Enum.sort() == [a: 1, b: 2] assert Enum.into(1..3, []) == [1, 2, 3] assert Enum.into(["H", "i"], "") == "Hi" end @@ -1430,7 +1430,7 @@ defmodule EnumTest do test "unzip/1" do assert Enum.unzip([{:a, 1}, {:b, 2}, {:c, 3}]) == {[:a, :b, :c], [1, 2, 3]} assert Enum.unzip([]) == {[], []} - assert Enum.unzip(%{a: 1, b: 2}) == {[:a, :b], [1, 2]} + assert Enum.unzip(%{a: 1}) == {[:a], [1]} assert Enum.unzip(foo: "a", bar: "b") == {[:foo, :bar], ["a", "b"]} assert_raise FunctionClauseError, fn -> Enum.unzip([{:a, 1}, {:b, 2, "foo"}]) end diff --git a/lib/elixir/test/elixir/float_test.exs b/lib/elixir/test/elixir/float_test.exs index 34e991659cb..e56c3b991ae 100644 --- a/lib/elixir/test/elixir/float_test.exs +++ b/lib/elixir/test/elixir/float_test.exs @@ -104,9 +104,9 @@ defmodule FloatTest do assert Float.ceil(7.5432e3) === 7544.0 assert Float.ceil(7.5e-3) === 1.0 assert Float.ceil(-12.32453e4) === -123_245.0 - assert Float.ceil(-12.32453e-10) === 0.0 + assert Float.ceil(-12.32453e-10) === -0.0 assert Float.ceil(0.32453e-10) === 1.0 - assert Float.ceil(-0.32453e-10) === 0.0 + assert Float.ceil(-0.32453e-10) === -0.0 assert Float.ceil(1.32453e-10) === 1.0 assert Float.ceil(0.0) === 0.0 end @@ -130,7 +130,7 @@ defmodule FloatTest do assert Float.ceil(-12.524235, 3) === -12.524 assert Float.ceil(12.32453e-20, 2) === 0.01 - assert Float.ceil(-12.32453e-20, 2) === 0.0 + assert Float.ceil(-12.32453e-20, 2) === -0.0 assert Float.ceil(0.0, 2) === 0.0 @@ -139,6 +139,11 @@ defmodule FloatTest do end end + test "with small floats rounded up to -0.0" do + assert Float.ceil(-0.1, 0) === -0.0 + assert Float.ceil(-0.01, 1) === -0.0 + end + test "with subnormal floats" do assert Float.ceil(5.0e-324, 0) === 1.0 assert Float.ceil(5.0e-324, 1) === 0.1 @@ -172,6 +177,17 @@ defmodule FloatTest do end end + test "with small floats rounded to +0.0 / -0.0" do + assert Float.round(0.01, 0) === 0.0 + assert Float.round(0.01, 1) === 0.0 + + assert Float.round(-0.01, 0) === -0.0 + assert Float.round(-0.01, 1) === -0.0 + + assert Float.round(-0.49999, 0) === -0.0 + assert Float.round(-0.049999, 1) === -0.0 + end + test "with subnormal floats" do for precision <- 0..15 do assert Float.round(5.0e-324, precision) === 0.0 From e520e8ce966e74170a13d7ca90b55b0cc27dda04 Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Fri, 23 Feb 2024 23:31:01 +0900 Subject: [PATCH 152/185] Fix unexpected rounding signs on OPT26- (#13365) --- .formatter.exs | 5 +- lib/elixir/lib/float.ex | 11 +- lib/elixir/test/elixir/float_test.exs | 155 ++++++++++++++------------ 3 files changed, 95 insertions(+), 76 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index f9bb2b1f0f2..b09e896884e 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -13,7 +13,10 @@ assert_same: 2, # Errors tests - assert_eval_raise: 3 + assert_eval_raise: 3, + + # Float tests + float_assert: 1 ], normalize_bitstring_modifiers: false ] diff --git a/lib/elixir/lib/float.ex b/lib/elixir/lib/float.ex index 63fd3a55548..b0bf7697172 100644 --- a/lib/elixir/lib/float.ex +++ b/lib/elixir/lib/float.ex @@ -374,8 +374,8 @@ defmodule Float do case rounding do :ceil when sign === 0 -> 1 / power_of_10(precision) :floor when sign === 1 -> -1 / power_of_10(precision) - :ceil when sign === 1 -> -0.0 - :half_up when sign === 1 -> -0.0 + :ceil when sign === 1 -> minus_zero() + :half_up when sign === 1 -> minus_zero() _ -> 0.0 end @@ -406,7 +406,7 @@ defmodule Float do cond do num == 0 and sign == 1 -> - -0.0 + minus_zero() num == 0 -> 0.0 @@ -422,6 +422,11 @@ defmodule Float do end end + # TODO remove once we require Erlang/OTP 27+ + # This function tricks the compiler to avoid this bug in previous versions: + # https://github.com/elixir-lang/elixir/blob/main/lib/elixir/lib/float.ex#L408-L412 + defp minus_zero, do: -0.0 + defp decompose(significant, initial) do decompose(significant, 1, 0, initial) end diff --git a/lib/elixir/test/elixir/float_test.exs b/lib/elixir/test/elixir/float_test.exs index e56c3b991ae..cc4c96c0359 100644 --- a/lib/elixir/test/elixir/float_test.exs +++ b/lib/elixir/test/elixir/float_test.exs @@ -5,6 +5,17 @@ defmodule FloatTest do doctest Float + # TODO remove and replace by assert once we require Erlang/OTP 27+ + # We can't easily distinguish between -0.0 and +0.0 on previous version + defmacrop float_assert({:===, _, [left, right]}) do + quote do + # note: these are pure functions so no need to use bind_quoted + # we favor a useful error message instead + assert unquote(left) === unquote(right) + assert to_string(unquote(left)) === to_string(unquote(right)) + end + end + test "parse/1" do assert Float.parse("12") === {12.0, ""} assert Float.parse("-12") === {-12.0, ""} @@ -45,39 +56,39 @@ defmodule FloatTest do end test "floor/1" do - assert Float.floor(12.524235) === 12.0 - assert Float.floor(-12.5) === -13.0 - assert Float.floor(-12.524235) === -13.0 - assert Float.floor(7.5e3) === 7500.0 - assert Float.floor(7.5432e3) === 7543.0 - assert Float.floor(7.5e-3) === 0.0 - assert Float.floor(-12.32453e4) === -123_246.0 - assert Float.floor(-12.32453e-10) === -1.0 - assert Float.floor(0.32453e-10) === 0.0 - assert Float.floor(-0.32453e-10) === -1.0 - assert Float.floor(1.32453e-10) === 0.0 + float_assert Float.floor(12.524235) === 12.0 + float_assert Float.floor(-12.5) === -13.0 + float_assert Float.floor(-12.524235) === -13.0 + float_assert Float.floor(7.5e3) === 7500.0 + float_assert Float.floor(7.5432e3) === 7543.0 + float_assert Float.floor(7.5e-3) === 0.0 + float_assert Float.floor(-12.32453e4) === -123_246.0 + float_assert Float.floor(-12.32453e-10) === -1.0 + float_assert Float.floor(0.32453e-10) === 0.0 + float_assert Float.floor(-0.32453e-10) === -1.0 + float_assert Float.floor(1.32453e-10) === 0.0 end describe "floor/2" do test "with 0.0" do for precision <- 0..15 do - assert Float.floor(0.0, precision) === 0.0 - assert Float.floor(-0.0, precision) === -0.0 + float_assert Float.floor(0.0, precision) === 0.0 + float_assert Float.floor(-0.0, precision) === -0.0 end end test "floor/2 with precision" do - assert Float.floor(12.524235, 0) === 12.0 - assert Float.floor(-12.524235, 0) === -13.0 + float_assert Float.floor(12.524235, 0) === 12.0 + float_assert Float.floor(-12.524235, 0) === -13.0 - assert Float.floor(12.52, 2) === 12.51 - assert Float.floor(-12.52, 2) === -12.52 + float_assert Float.floor(12.52, 2) === 12.51 + float_assert Float.floor(-12.52, 2) === -12.52 - assert Float.floor(12.524235, 2) === 12.52 - assert Float.floor(-12.524235, 3) === -12.525 + float_assert Float.floor(12.524235, 2) === 12.52 + float_assert Float.floor(-12.524235, 3) === -12.525 - assert Float.floor(12.32453e-20, 2) === 0.0 - assert Float.floor(-12.32453e-20, 2) === -0.01 + float_assert Float.floor(12.32453e-20, 2) === 0.0 + float_assert Float.floor(-12.32453e-20, 2) === -0.01 assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> Float.floor(1.1, 16) @@ -85,54 +96,54 @@ defmodule FloatTest do end test "with subnormal floats" do - assert Float.floor(-5.0e-324, 0) === -1.0 - assert Float.floor(-5.0e-324, 1) === -0.1 - assert Float.floor(-5.0e-324, 2) === -0.01 - assert Float.floor(-5.0e-324, 15) === -0.000000000000001 + float_assert Float.floor(-5.0e-324, 0) === -1.0 + float_assert Float.floor(-5.0e-324, 1) === -0.1 + float_assert Float.floor(-5.0e-324, 2) === -0.01 + float_assert Float.floor(-5.0e-324, 15) === -0.000000000000001 for precision <- 0..15 do - assert Float.floor(5.0e-324, precision) === 0.0 + float_assert Float.floor(5.0e-324, precision) === 0.0 end end end test "ceil/1" do - assert Float.ceil(12.524235) === 13.0 - assert Float.ceil(-12.5) === -12.0 - assert Float.ceil(-12.524235) === -12.0 - assert Float.ceil(7.5e3) === 7500.0 - assert Float.ceil(7.5432e3) === 7544.0 - assert Float.ceil(7.5e-3) === 1.0 - assert Float.ceil(-12.32453e4) === -123_245.0 - assert Float.ceil(-12.32453e-10) === -0.0 - assert Float.ceil(0.32453e-10) === 1.0 - assert Float.ceil(-0.32453e-10) === -0.0 - assert Float.ceil(1.32453e-10) === 1.0 - assert Float.ceil(0.0) === 0.0 + float_assert Float.ceil(12.524235) === 13.0 + float_assert Float.ceil(-12.5) === -12.0 + float_assert Float.ceil(-12.524235) === -12.0 + float_assert Float.ceil(7.5e3) === 7500.0 + float_assert Float.ceil(7.5432e3) === 7544.0 + float_assert Float.ceil(7.5e-3) === 1.0 + float_assert Float.ceil(-12.32453e4) === -123_245.0 + float_assert Float.ceil(-12.32453e-10) === -0.0 + float_assert Float.ceil(0.32453e-10) === 1.0 + float_assert Float.ceil(-0.32453e-10) === -0.0 + float_assert Float.ceil(1.32453e-10) === 1.0 + float_assert Float.ceil(0.0) === 0.0 end describe "ceil/2" do test "with 0.0" do for precision <- 0..15 do - assert Float.ceil(0.0, precision) === 0.0 - assert Float.ceil(-0.0, precision) === -0.0 + float_assert Float.ceil(0.0, precision) === 0.0 + float_assert Float.ceil(-0.0, precision) === -0.0 end end test "with regular floats" do - assert Float.ceil(12.524235, 0) === 13.0 - assert Float.ceil(-12.524235, 0) === -12.0 + float_assert Float.ceil(12.524235, 0) === 13.0 + float_assert Float.ceil(-12.524235, 0) === -12.0 - assert Float.ceil(12.52, 2) === 12.52 - assert Float.ceil(-12.52, 2) === -12.51 + float_assert Float.ceil(12.52, 2) === 12.52 + float_assert Float.ceil(-12.52, 2) === -12.51 - assert Float.ceil(12.524235, 2) === 12.53 - assert Float.ceil(-12.524235, 3) === -12.524 + float_assert Float.ceil(12.524235, 2) === 12.53 + float_assert Float.ceil(-12.524235, 3) === -12.524 - assert Float.ceil(12.32453e-20, 2) === 0.01 - assert Float.ceil(-12.32453e-20, 2) === -0.0 + float_assert Float.ceil(12.32453e-20, 2) === 0.01 + float_assert Float.ceil(-12.32453e-20, 2) === -0.0 - assert Float.ceil(0.0, 2) === 0.0 + float_assert Float.ceil(0.0, 2) === 0.0 assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> Float.ceil(1.1, 16) @@ -140,18 +151,18 @@ defmodule FloatTest do end test "with small floats rounded up to -0.0" do - assert Float.ceil(-0.1, 0) === -0.0 - assert Float.ceil(-0.01, 1) === -0.0 + float_assert Float.ceil(-0.1, 0) === -0.0 + float_assert Float.ceil(-0.01, 1) === -0.0 end test "with subnormal floats" do - assert Float.ceil(5.0e-324, 0) === 1.0 - assert Float.ceil(5.0e-324, 1) === 0.1 - assert Float.ceil(5.0e-324, 2) === 0.01 - assert Float.ceil(5.0e-324, 15) === 0.000000000000001 + float_assert Float.ceil(5.0e-324, 0) === 1.0 + float_assert Float.ceil(5.0e-324, 1) === 0.1 + float_assert Float.ceil(5.0e-324, 2) === 0.01 + float_assert Float.ceil(5.0e-324, 15) === 0.000000000000001 for precision <- 0..15 do - assert Float.ceil(-5.0e-324, precision) === -0.0 + float_assert Float.ceil(-5.0e-324, precision) === -0.0 end end end @@ -159,18 +170,18 @@ defmodule FloatTest do describe "round/2" do test "with 0.0" do for precision <- 0..15 do - assert Float.round(0.0, precision) === 0.0 - assert Float.round(-0.0, precision) === -0.0 + float_assert Float.round(0.0, precision) === 0.0 + float_assert Float.round(-0.0, precision) === -0.0 end end test "with regular floats" do - assert Float.round(5.5675, 3) === 5.567 - assert Float.round(-5.5674, 3) === -5.567 - assert Float.round(5.5, 3) === 5.5 - assert Float.round(5.5e-10, 10) === 5.0e-10 - assert Float.round(5.5e-10, 8) === 0.0 - assert Float.round(5.0, 0) === 5.0 + float_assert Float.round(5.5675, 3) === 5.567 + float_assert Float.round(-5.5674, 3) === -5.567 + float_assert Float.round(5.5, 3) === 5.5 + float_assert Float.round(5.5e-10, 10) === 5.0e-10 + float_assert Float.round(5.5e-10, 8) === 0.0 + float_assert Float.round(5.0, 0) === 5.0 assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> Float.round(1.1, 16) @@ -178,20 +189,20 @@ defmodule FloatTest do end test "with small floats rounded to +0.0 / -0.0" do - assert Float.round(0.01, 0) === 0.0 - assert Float.round(0.01, 1) === 0.0 + float_assert Float.round(0.01, 0) === 0.0 + float_assert Float.round(0.01, 1) === 0.0 - assert Float.round(-0.01, 0) === -0.0 - assert Float.round(-0.01, 1) === -0.0 + float_assert Float.round(-0.01, 0) === -0.0 + float_assert Float.round(-0.01, 1) === -0.0 - assert Float.round(-0.49999, 0) === -0.0 - assert Float.round(-0.049999, 1) === -0.0 + float_assert Float.round(-0.49999, 0) === -0.0 + float_assert Float.round(-0.049999, 1) === -0.0 end test "with subnormal floats" do for precision <- 0..15 do - assert Float.round(5.0e-324, precision) === 0.0 - assert Float.round(-5.0e-324, precision) === -0.0 + float_assert Float.round(5.0e-324, precision) === 0.0 + float_assert Float.round(-5.0e-324, precision) === -0.0 end end end From fe16fd34428597d0a3e35da21a89f7d69200c60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Samson?= Date: Sat, 24 Feb 2024 16:23:05 +0100 Subject: [PATCH 153/185] Add documentation to `Exception` callbacks (#13367) --- lib/elixir/lib/exception.ex | 14 ++++++++++++++ lib/iex/test/iex/helpers_test.exs | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index fe0f45ba018..6575743c889 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -31,7 +31,21 @@ defmodule Exception do @type arity_or_args :: non_neg_integer | list @type location :: keyword + @doc """ + Receives the arguments given to `raise/2` and returns the exception struct. + + The default implementation accepts either a set of keyword arguments + that is merged into the struct or a string to be used as the exception's message. + """ @callback exception(term) :: t + + @doc """ + Receives the exception struct and must return its message. + + Most commonly exceptions have a message field which by default is accessed + by this function. However, if an exception does not have a message field, + this function must be explicitly implemented. + """ @callback message(t) :: String.t() @doc """ diff --git a/lib/iex/test/iex/helpers_test.exs b/lib/iex/test/iex/helpers_test.exs index 54f946516ce..d2580eeb297 100644 --- a/lib/iex/test/iex/helpers_test.exs +++ b/lib/iex/test/iex/helpers_test.exs @@ -802,7 +802,7 @@ defmodule IEx.HelpersTest do assert capture_io(fn -> b(NoMix.run()) end) == "Could not load module NoMix, got: nofile\n" - assert capture_io(fn -> b(Exception.message() / 1) end) == + assert capture_io(fn -> b(Exception.message() / 1) end) =~ "@callback message(t()) :: String.t()\n\n" assert capture_io(fn -> b(:gen_server.handle_cast() / 2) end) =~ From 097c4cf672cd8e6810c7d7adfe9a29c182884e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 26 Feb 2024 18:28:35 +0100 Subject: [PATCH 154/185] Emit defmodule tracing event --- lib/elixir/lib/code.ex | 4 ++++ lib/elixir/src/elixir_module.erl | 1 + lib/elixir/test/elixir/kernel/tracers_test.exs | 13 +++++++++++++ 3 files changed, 18 insertions(+) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index de1a9fa9c87..e8ce423a1b3 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -158,6 +158,10 @@ defmodule Code do of keys to traverse in the application environment and `return` is either `{:ok, value}` or `:error`. + * `:defmodule` - (since v1.16.2) traced as soon as the definition of a module + starts. This is invoked early on in the module life-cycle, `Module.open?/1` + still returns `false` for such traces + * `{:on_module, bytecode, _ignore}` - (since v1.13.0) traced whenever a module is defined. This is equivalent to the `@after_compile` callback and invoked after any `@after_compile` in the given module. The third element is currently diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index 637598d6f27..5f8d066ee44 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -118,6 +118,7 @@ invalid_module_name(Module) -> compile(Line, Module, ModuleAsCharlist, Block, Vars, Prune, E) -> File = ?key(E, file), check_module_availability(Module, Line, E), + elixir_env:trace(defmodule, E), CompilerModules = compiler_modules(), {Tables, Ref} = build(Module, Line, File, E), diff --git a/lib/elixir/test/elixir/kernel/tracers_test.exs b/lib/elixir/test/elixir/kernel/tracers_test.exs index 7374ea41efd..ee282c10261 100644 --- a/lib/elixir/test/elixir/kernel/tracers_test.exs +++ b/lib/elixir/test/elixir/kernel/tracers_test.exs @@ -201,6 +201,19 @@ defmodule Kernel.TracersTest do end """) + assert_receive {:defmodule, %{module: Sample, function: nil}} + assert_receive {{:on_module, <<_::binary>>, :none}, %{module: Sample, function: nil}} + after + :code.purge(Sample) + :code.delete(Sample) + end + + test "traces dynamic modules" do + compile_string(""" + Module.create(Sample, :ok, __ENV__) + """) + + assert_receive {:defmodule, %{module: Sample, function: nil}} assert_receive {{:on_module, <<_::binary>>, :none}, %{module: Sample, function: nil}} after :code.purge(Sample) From 325877a224ab5cb1789843118de1c87ef30a5b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 26 Feb 2024 20:12:21 +0100 Subject: [PATCH 155/185] Clarify the meaning of --overwrite, closes #13371 --- lib/mix/lib/mix/tasks/release.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/lib/mix/tasks/release.ex b/lib/mix/lib/mix/tasks/release.ex index e9d13d58f08..4db846e732c 100644 --- a/lib/mix/lib/mix/tasks/release.ex +++ b/lib/mix/lib/mix/tasks/release.ex @@ -1021,7 +1021,7 @@ defmodule Mix.Tasks.Release do * `--no-deps-check` - does not check dependencies * `--no-elixir-version-check` - does not check Elixir version * `--no-compile` - does not compile before assembling the release - * `--overwrite` - if there is an existing release version, overwrite it + * `--overwrite` - overwrite existing files instead of prompting the user for action * `--path` - the path of the release * `--quiet` - does not write progress to the standard output * `--version` - the version of the release From 079b92e98e7da1dd7b381a996803c812f49071db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 29 Feb 2024 08:38:29 +0100 Subject: [PATCH 156/185] Support recompiling local Mix.install/2 dependencies (#13375) --- lib/iex/lib/iex/helpers.ex | 64 ++++++++++++++++++++++++------------- lib/mix/lib/mix.ex | 65 +++++++++++++++++++++++++++++--------- lib/mix/test/mix_test.exs | 36 +++++++++++++++++++-- 3 files changed, 126 insertions(+), 39 deletions(-) diff --git a/lib/iex/lib/iex/helpers.ex b/lib/iex/lib/iex/helpers.ex index fa498735621..3e56a49f0cf 100644 --- a/lib/iex/lib/iex/helpers.ex +++ b/lib/iex/lib/iex/helpers.ex @@ -69,17 +69,22 @@ defmodule IEx.Helpers do import IEx, only: [dont_display_result: 0] @doc """ - Recompiles the current Mix project. + Recompiles the current Mix project or Mix install + dependencies. - This helper only works when IEx is started with a Mix - project, for example, `iex -S mix`. Note this function - simply recompiles Elixir modules, without reloading - configuration, recompiling dependencies, or restarting - applications. + This helper requires either `Mix.install/2` to have been + called within the current IEx session or for IEx to be + started alongside, for example, `iex -S mix`. - Therefore, any long running process may crash on recompilation, - as changed modules will be temporarily removed and recompiled, - without going through the proper code change callback. + In the `Mix.install/1` case, it will recompile any outdated + path dependency declared during install. Within a project, + it will recompile any outdated module. + + Note this function simply recompiles Elixir modules, without + reloading configuration or restarting applications. This means + any long running process may crash on recompilation, as changed + modules will be temporarily removed and recompiled, without + going through the proper code change callback. If you want to reload a single module, consider using `r(ModuleName)` instead. @@ -93,21 +98,36 @@ defmodule IEx.Helpers do """ def recompile(options \\ []) do - if mix_started?() do - config = Mix.Project.config() - consolidation = Mix.Project.consolidation_path(config) - reenable_tasks(config) + cond do + not mix_started?() -> + IO.puts(IEx.color(:eval_error, "Mix is not running. Please start IEx with: iex -S mix")) + :error + + Mix.installed?() -> + Mix.in_install_project(fn -> + do_recompile(options) + # Just as with Mix.install/2 we clear all task invocations, + # so that we can recompile the dependencies again next time + Mix.Task.clear() + :ok + end) + + true -> + do_recompile(options) + end + end - force? = Keyword.get(options, :force, false) - args = ["--purge-consolidation-path-if-stale", "--return-errors", consolidation] - args = if force?, do: ["--force" | args], else: args + defp do_recompile(options) do + config = Mix.Project.config() + consolidation = Mix.Project.consolidation_path(config) + reenable_tasks(config) - {result, _} = Mix.Task.run("compile", args) - result - else - IO.puts(IEx.color(:eval_error, "Mix is not running. Please start IEx with: iex -S mix")) - :error - end + force? = Keyword.get(options, :force, false) + args = ["--purge-consolidation-path-if-stale", "--return-errors", consolidation] + args = if force?, do: ["--force" | args], else: args + + {result, _} = Mix.Task.run("compile", args) + result end defp mix_started? do diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index bf17b11565b..e98f514bdf4 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -858,23 +858,14 @@ defmodule Mix do File.rm_rf!(install_dir) end - config = [ - version: "0.1.0", - build_embedded: false, - build_per_environment: true, - build_path: "_build", - lockfile: "mix.lock", - deps_path: "deps", + dynamic_config = [ deps: deps, - app: :mix_install, - erlc_paths: [], - elixirc_paths: [], - compilers: [], consolidate_protocols: consolidate_protocols?, - config_path: config_path, - prune_code_paths: false + config_path: config_path ] + config = install_project_config(dynamic_config) + started_apps = Application.started_applications() :ok = Mix.ProjectStack.push(@mix_install_project, config, "nofile") build_dir = Path.join(install_dir, "_build") @@ -940,13 +931,18 @@ defmodule Mix do end end - Mix.State.put(:installed, id) + Mix.State.put(:installed, {id, dynamic_config}) :ok after Mix.ProjectStack.pop() + # Clear all tasks invoked during installation, since there + # is no reason to keep this in memory. Additionally this + # allows us to rerun tasks for the dependencies later on, + # such as recompilation + Mix.Task.clear() end - ^id when not force? -> + {^id, _dynamic_config} when not force? -> :ok _ -> @@ -981,6 +977,45 @@ defmodule Mix do Path.join([install_root, version, cache_id]) end + defp install_project_config(dynamic_config) do + [ + version: "0.1.0", + build_embedded: false, + build_per_environment: true, + build_path: "_build", + lockfile: "mix.lock", + deps_path: "deps", + app: :mix_install, + erlc_paths: [], + elixirc_paths: [], + compilers: [], + prune_code_paths: false + ] ++ dynamic_config + end + + @doc false + def in_install_project(fun) do + case Mix.State.get(:installed) do + {id, dynamic_config} -> + config = install_project_config(dynamic_config) + + install_dir = install_dir(id) + + File.cd!(install_dir, fn -> + :ok = Mix.ProjectStack.push(@mix_install_project, config, "nofile") + + try do + fun.() + after + Mix.ProjectStack.pop() + end + end) + + nil -> + Mix.raise("trying to call Mix.in_install_project/1, but Mix.install/2 was never called") + end + end + @doc """ Returns whether `Mix.install/2` was called in the current node. """ diff --git a/lib/mix/test/mix_test.exs b/lib/mix/test/mix_test.exs index e7b02da1e7d..618d4cba679 100644 --- a/lib/mix/test/mix_test.exs +++ b/lib/mix/test/mix_test.exs @@ -39,7 +39,7 @@ defmodule MixTest do assert Protocol.consolidated?(InstallTest.Protocol) assert_received {:mix_shell, :info, ["==> install_test"]} - assert_received {:mix_shell, :info, ["Compiling 1 file (.ex)"]} + assert_received {:mix_shell, :info, ["Compiling 2 files (.ex)"]} assert_received {:mix_shell, :info, ["Generated install_test app"]} refute_received _ @@ -67,7 +67,7 @@ defmodule MixTest do assert File.dir?(Path.join(tmp_dir, "installs")) assert_received {:mix_shell, :info, ["==> install_test"]} - assert_received {:mix_shell, :info, ["Compiling 1 file (.ex)"]} + assert_received {:mix_shell, :info, ["Compiling 2 files (.ex)"]} assert_received {:mix_shell, :info, ["Generated install_test app"]} refute_received _ @@ -345,6 +345,36 @@ defmodule MixTest do assert Mix.installed?() end + test "in_install_project", %{tmp_dir: tmp_dir} do + Mix.install([ + {:install_test, path: Path.join(tmp_dir, "install_test")} + ]) + + Mix.in_install_project(fn -> + config = Mix.Project.config() + assert [{:install_test, [path: _]}] = config[:deps] + end) + end + + test "in_install_project recompile", %{tmp_dir: tmp_dir} do + Mix.install([ + {:install_test, path: Path.join(tmp_dir, "install_test")} + ]) + + File.write!("#{tmp_dir}/install_test/lib/install_test.ex", """ + defmodule InstallTest do + def hello do + :universe + end + end + """) + + Mix.in_install_project(fn -> + Mix.Task.run("compile") + assert apply(InstallTest, :hello, []) == :universe + end) + end + defp test_project(%{tmp_dir: tmp_dir}) do path = :code.get_path() @@ -384,7 +414,9 @@ defmodule MixTest do :world end end + """) + File.write!("#{tmp_dir}/install_test/lib/install_test_protocol.ex", """ defprotocol InstallTest.Protocol do def foo(x) end From e78538a42bbd19b3f0ccc48e1efda32b96939cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 29 Feb 2024 19:19:12 +0100 Subject: [PATCH 157/185] Add environment variable for reusing Mix.install/2 installation (#13378) --- lib/iex/lib/iex/helpers.ex | 8 ++ lib/mix/lib/mix.ex | 75 +++++++++++++++---- lib/mix/test/mix_test.exs | 146 +++++++++++++++++++++++++++++-------- 3 files changed, 184 insertions(+), 45 deletions(-) diff --git a/lib/iex/lib/iex/helpers.ex b/lib/iex/lib/iex/helpers.ex index 3e56a49f0cf..8ba18e573ab 100644 --- a/lib/iex/lib/iex/helpers.ex +++ b/lib/iex/lib/iex/helpers.ex @@ -105,6 +105,14 @@ defmodule IEx.Helpers do Mix.installed?() -> Mix.in_install_project(fn -> + # TODO: remove this once Mix requires Hex with the fix from + # https://github.com/hexpm/hex/pull/1015 + # Context: Mix.install/1 starts :hex if necessary and stops + # it afterwards. Calling compile here may require hex to be + # started and that should happen automatically, but because + # of a bug it is not (fixed in the linked PR). + _ = Application.ensure_all_started(:hex) + do_recompile(options) # Just as with Mix.install/2 we clear all task invocations, # so that we can recompile the dependencies again next time diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index e98f514bdf4..3a3e2dd1d4c 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -661,8 +661,12 @@ defmodule Mix do This function can only be called outside of a Mix project and only with the same dependencies in the given VM. - **Note:** this feature is currently experimental and it may change - in future releases. + The `MIX_INSTALL_RESTORE_PROJECT_DIR` environment variable may be specified. + It should point to a previous installation directory, which can be obtained + with `Mix.install_project_dir/0` (after calling `Mix.install/2`). Using a + restore dir may speed up the installation, since matching dependencies do + not need be refetched nor recompiled. This environment variable is ignored + if `:force` is enabled. ## Options @@ -848,14 +852,14 @@ defmodule Mix do Application.put_all_env(config, persistent: true) System.put_env(system_env) - install_dir = install_dir(id) + install_project_dir = install_project_dir(id) if Keyword.fetch!(opts, :verbose) do - Mix.shell().info("Mix.install/2 using #{install_dir}") + Mix.shell().info("Mix.install/2 using #{install_project_dir}") end if force? do - File.rm_rf!(install_dir) + File.rm_rf!(install_project_dir) end dynamic_config = [ @@ -868,21 +872,28 @@ defmodule Mix do started_apps = Application.started_applications() :ok = Mix.ProjectStack.push(@mix_install_project, config, "nofile") - build_dir = Path.join(install_dir, "_build") + build_dir = Path.join(install_project_dir, "_build") external_lockfile = expand_path(opts[:lockfile], deps, :lockfile, "mix.lock") try do first_build? = not File.dir?(build_dir) - File.mkdir_p!(install_dir) - File.cd!(install_dir, fn -> + restore_dir = System.get_env("MIX_INSTALL_RESTORE_PROJECT_DIR") + + if first_build? and restore_dir != nil and not force? do + File.cp_r(restore_dir, install_project_dir) + end + + File.mkdir_p!(install_project_dir) + + File.cd!(install_project_dir, fn -> if config_path do Mix.Task.rerun("loadconfig") end cond do external_lockfile -> - md5_path = Path.join(install_dir, "merge.lock.md5") + md5_path = Path.join(install_project_dir, "merge.lock.md5") old_md5 = case File.read(md5_path) do @@ -893,7 +904,7 @@ defmodule Mix do new_md5 = external_lockfile |> File.read!() |> :erlang.md5() if old_md5 != new_md5 do - lockfile = Path.join(install_dir, "mix.lock") + lockfile = Path.join(install_project_dir, "mix.lock") old_lock = Mix.Dep.Lock.read(lockfile) new_lock = Mix.Dep.Lock.read(external_lockfile) Mix.Dep.Lock.write(Map.merge(old_lock, new_lock), file: lockfile) @@ -931,6 +942,10 @@ defmodule Mix do end end + if restore_dir do + remove_leftover_deps(install_project_dir) + end + Mix.State.put(:installed, {id, dynamic_config}) :ok after @@ -968,7 +983,29 @@ defmodule Mix do Path.join(app_dir, relative_path) end - defp install_dir(cache_id) do + defp remove_leftover_deps(install_project_dir) do + build_lib_dir = Path.join([install_project_dir, "_build", "dev", "lib"]) + deps_dir = Path.join(install_project_dir, "deps") + + deps = File.ls!(build_lib_dir) + + loaded_deps = + for {app, _description, _version} <- Application.loaded_applications(), + into: MapSet.new(), + do: Atom.to_string(app) + + # We want to keep :mix_install, but it has no application + loaded_deps = MapSet.put(loaded_deps, "mix_install") + + for dep <- deps, not MapSet.member?(loaded_deps, dep) do + build_path = Path.join(build_lib_dir, dep) + File.rm_rf(build_path) + dep_path = Path.join(deps_dir, dep) + File.rm_rf(dep_path) + end + end + + defp install_project_dir(cache_id) do install_root = System.get_env("MIX_INSTALL_DIR") || Path.join(Mix.Utils.mix_cache(), "installs") @@ -999,9 +1036,9 @@ defmodule Mix do {id, dynamic_config} -> config = install_project_config(dynamic_config) - install_dir = install_dir(id) + install_project_dir = install_project_dir(id) - File.cd!(install_dir, fn -> + File.cd!(install_project_dir, fn -> :ok = Mix.ProjectStack.push(@mix_install_project, config, "nofile") try do @@ -1016,6 +1053,18 @@ defmodule Mix do end end + @doc """ + Returns the directory where the current `Mix.install/2` project + resides. + """ + @spec install_project_dir() :: Path.t() + def install_project_dir() do + case Mix.State.get(:installed) do + {id, _dynamic_config} -> install_project_dir(id) + nil -> nil + end + end + @doc """ Returns whether `Mix.install/2` was called in the current node. """ diff --git a/lib/mix/test/mix_test.exs b/lib/mix/test/mix_test.exs index 618d4cba679..266533fcdba 100644 --- a/lib/mix/test/mix_test.exs +++ b/lib/mix/test/mix_test.exs @@ -263,30 +263,26 @@ defmodule MixTest do [ {:git_repo, git: fixture_path("git_repo")} ], - lockfile: lockfile, - verbose: true + lockfile: lockfile ) assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]} - assert_received {:mix_shell, :info, ["Mix.install/2 using " <> install_dir]} - assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev - after - purge([GitRepo, GitRepo.MixProject]) + + install_project_dir = Mix.install_project_dir() + assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev end test ":lockfile merging", %{tmp_dir: tmp_dir} do [rev1, rev2 | _] = get_git_repo_revs("git_repo") - Mix.install( - [ - {:git_repo, git: fixture_path("git_repo")} - ], - verbose: true - ) + Mix.install([ + {:git_repo, git: fixture_path("git_repo")} + ]) assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]} - assert_received {:mix_shell, :info, ["Mix.install/2 using " <> install_dir]} - assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev1 + + install_project_dir = Mix.install_project_dir() + assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev1 Mix.Project.push(GitApp) lockfile = Path.join(tmp_dir, "lock") @@ -300,9 +296,7 @@ defmodule MixTest do lockfile: lockfile ) - assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev1 - after - purge([GitRepo, GitRepo.MixProject]) + assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev1 end test ":lockfile with application name", %{tmp_dir: tmp_dir} do @@ -318,15 +312,12 @@ defmodule MixTest do {:install_test, path: Path.join(tmp_dir, "install_test")}, {:git_repo, git: fixture_path("git_repo")} ], - lockfile: :install_test, - verbose: true + lockfile: :install_test ) assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]} - assert_received {:mix_shell, :info, ["Mix.install/2 using " <> install_dir]} - assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev - after - purge([GitRepo, GitRepo.MixProject]) + install_project_dir = Mix.install_project_dir() + assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev end test ":lockfile that does not exist" do @@ -335,6 +326,73 @@ defmodule MixTest do end end + test "restore dir", %{tmp_dir: tmp_dir} do + with_cleanup(fn -> + Mix.install([ + {:git_repo, git: fixture_path("git_repo")} + ]) + + assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]} + assert_received {:mix_shell, :info, ["==> git_repo"]} + assert_received {:mix_shell, :info, ["Compiling 1 file (.ex)"]} + assert_received {:mix_shell, :info, ["Generated git_repo app"]} + refute_received _ + + install_project_dir = Mix.install_project_dir() + build_lib_path = Path.join([install_project_dir, "_build", "dev", "lib"]) + deps_path = Path.join([install_project_dir, "deps"]) + + assert File.ls!(build_lib_path) |> Enum.sort() == ["git_repo", "mix_install"] + assert File.ls!(deps_path) == ["git_repo"] + + System.put_env("MIX_INSTALL_RESTORE_PROJECT_DIR", install_project_dir) + end) + + # Adding a dependency + + with_cleanup(fn -> + Mix.install([ + {:git_repo, git: fixture_path("git_repo")}, + {:install_test, path: Path.join(tmp_dir, "install_test")} + ]) + + assert_received {:mix_shell, :info, ["==> install_test"]} + assert_received {:mix_shell, :info, ["Compiling 2 files (.ex)"]} + assert_received {:mix_shell, :info, ["Generated install_test app"]} + refute_received _ + + install_project_dir = Mix.install_project_dir() + build_lib_path = Path.join([install_project_dir, "_build", "dev", "lib"]) + deps_path = Path.join([install_project_dir, "deps"]) + + assert File.ls!(build_lib_path) |> Enum.sort() == + ["git_repo", "install_test", "mix_install"] + + assert File.ls!(deps_path) == ["git_repo"] + + System.put_env("MIX_INSTALL_RESTORE_PROJECT_DIR", install_project_dir) + end) + + # Removing a dependency + + with_cleanup(fn -> + Mix.install([ + {:install_test, path: Path.join(tmp_dir, "install_test")} + ]) + + refute_received _ + + install_project_dir = Mix.install_project_dir() + build_lib_path = Path.join([install_project_dir, "_build", "dev", "lib"]) + deps_path = Path.join([install_project_dir, "deps"]) + + assert File.ls!(build_lib_path) |> Enum.sort() == ["install_test", "mix_install"] + assert File.ls!(deps_path) == [] + end) + after + System.delete_env("MIX_INSTALL_RESTORE_PROJECT_DIR") + end + test "installed?", %{tmp_dir: tmp_dir} do refute Mix.installed?() @@ -380,15 +438,7 @@ defmodule MixTest do on_exit(fn -> :code.set_path(path) - purge([InstallTest, InstallTest.MixProject, InstallTest.Protocol]) - - ExUnit.CaptureLog.capture_log(fn -> - Application.stop(:git_repo) - Application.unload(:git_repo) - - Application.stop(:install_test) - Application.unload(:install_test) - end) + cleanup_deps() end) Mix.State.put(:installed, nil) @@ -424,5 +474,37 @@ defmodule MixTest do [tmp_dir: tmp_dir] end + + defp with_cleanup(fun) do + path = :code.get_path() + + try do + fun.() + after + :code.set_path(path) + cleanup_deps() + + Mix.State.clear_cache() + Mix.State.put(:installed, nil) + end + end + + defp cleanup_deps() do + purge([ + GitRepo, + GitRepo.MixProject, + InstallTest, + InstallTest.MixProject, + InstallTest.Protocol + ]) + + ExUnit.CaptureLog.capture_log(fn -> + Application.stop(:git_repo) + Application.unload(:git_repo) + + Application.stop(:install_test) + Application.unload(:install_test) + end) + end end end From 565475202679ae5af5f2bc2ce340163382f50eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Thu, 29 Feb 2024 20:30:48 +0100 Subject: [PATCH 158/185] Add @doc since to install_project_dir --- lib/mix/lib/mix.ex | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index 3a3e2dd1d4c..ba379a790d0 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -352,8 +352,6 @@ defmodule Mix do * `MIX_INSTALL_DIR` - (since v1.12.0) specifies directory where `Mix.install/2` keeps install cache - * `MIX_INSTALL_FORCE` - (since v1.13.0) runs `Mix.install/2` with empty install cache - * `MIX_PATH` - appends extra code paths * `MIX_PROFILE` - a list of comma-separated Mix tasks to profile the time spent on @@ -661,13 +659,6 @@ defmodule Mix do This function can only be called outside of a Mix project and only with the same dependencies in the given VM. - The `MIX_INSTALL_RESTORE_PROJECT_DIR` environment variable may be specified. - It should point to a previous installation directory, which can be obtained - with `Mix.install_project_dir/0` (after calling `Mix.install/2`). Using a - restore dir may speed up the installation, since matching dependencies do - not need be refetched nor recompiled. This environment variable is ignored - if `:force` is enabled. - ## Options * `:force` - if `true`, runs with empty install cache. This is useful when you want @@ -783,6 +774,21 @@ defmodule Mix do The contents inside `defmodule` will only be expanded and executed after `Mix.install/2` runs, which means that any struct, macros, and imports will be correctly handled. + + ## Environment variables + + The `MIX_INSTALL_DIR` environment variable configures the directory that + caches all `Mix.install/2`. + + The `MIX_INSTALL_FORCE` is available since Elixir v1.13.0 and forces + `Mix.install/2` to discard any previously cached entry of the current install. + + The `MIX_INSTALL_RESTORE_PROJECT_DIR` environment variable may be specified + since Elixir v1.16.2. It should point to a previous installation directory, + which can be obtained with `Mix.install_project_dir/0` (after calling `Mix.install/2`). + Using a restore dir may speed up the installation, since matching dependencies + do not need be refetched nor recompiled. This environment variable is ignored + if `:force` is enabled. """ @doc since: "1.12.0" def install(deps, opts \\ []) @@ -1057,6 +1063,7 @@ defmodule Mix do Returns the directory where the current `Mix.install/2` project resides. """ + @doc since: "1.16.2" @spec install_project_dir() :: Path.t() def install_project_dir() do case Mix.State.get(:installed) do From 74ab38439e1179a16b8757d36786b53bfbd3213e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Fri, 1 Mar 2024 09:15:01 +0100 Subject: [PATCH 159/185] Fix Mix.install_project_dir/0 spec (#13381) --- lib/mix/lib/mix.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index ba379a790d0..61152f1c752 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -1064,7 +1064,7 @@ defmodule Mix do resides. """ @doc since: "1.16.2" - @spec install_project_dir() :: Path.t() + @spec install_project_dir() :: Path.t() | nil def install_project_dir() do case Mix.State.get(:installed) do {id, _dynamic_config} -> install_project_dir(id) From 29ef147a22ebc3ce823a444a0529ceada257a74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Fri, 1 Mar 2024 09:33:14 +0100 Subject: [PATCH 160/185] Remove consolidated when restoring Mix.install/2 dir (#13382) --- lib/mix/lib/mix.ex | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index 61152f1c752..7091a8b79a2 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -377,6 +377,8 @@ defmodule Mix do """ @mix_install_project __MODULE__.InstallProject + @mix_install_app :mix_install + @mix_install_app_string Atom.to_string(@mix_install_app) use Application @@ -888,6 +890,7 @@ defmodule Mix do if first_build? and restore_dir != nil and not force? do File.cp_r(restore_dir, install_project_dir) + remove_dep(install_project_dir, @mix_install_app_string) end File.mkdir_p!(install_project_dir) @@ -991,7 +994,6 @@ defmodule Mix do defp remove_leftover_deps(install_project_dir) do build_lib_dir = Path.join([install_project_dir, "_build", "dev", "lib"]) - deps_dir = Path.join(install_project_dir, "deps") deps = File.ls!(build_lib_dir) @@ -1001,16 +1003,23 @@ defmodule Mix do do: Atom.to_string(app) # We want to keep :mix_install, but it has no application - loaded_deps = MapSet.put(loaded_deps, "mix_install") + loaded_deps = MapSet.put(loaded_deps, @mix_install_app_string) for dep <- deps, not MapSet.member?(loaded_deps, dep) do - build_path = Path.join(build_lib_dir, dep) - File.rm_rf(build_path) - dep_path = Path.join(deps_dir, dep) - File.rm_rf(dep_path) + remove_dep(install_project_dir, dep) end end + defp remove_dep(install_project_dir, dep) do + build_lib_dir = Path.join([install_project_dir, "_build", "dev", "lib"]) + deps_dir = Path.join(install_project_dir, "deps") + + build_path = Path.join(build_lib_dir, dep) + File.rm_rf(build_path) + dep_path = Path.join(deps_dir, dep) + File.rm_rf(dep_path) + end + defp install_project_dir(cache_id) do install_root = System.get_env("MIX_INSTALL_DIR") || @@ -1028,7 +1037,7 @@ defmodule Mix do build_path: "_build", lockfile: "mix.lock", deps_path: "deps", - app: :mix_install, + app: @mix_install_app, erlc_paths: [], elixirc_paths: [], compilers: [], From e310dd91d68315176945c4e5cf9cf25ba830ddea Mon Sep 17 00:00:00 2001 From: H Date: Sun, 3 Mar 2024 22:41:28 -0800 Subject: [PATCH 161/185] Remove duplicate line in formatter_callback doc (#13387) --- lib/ex_unit/lib/ex_unit/formatter.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/formatter.ex b/lib/ex_unit/lib/ex_unit/formatter.ex index da41f6c3ef5..8f771cb0166 100644 --- a/lib/ex_unit/lib/ex_unit/formatter.ex +++ b/lib/ex_unit/lib/ex_unit/formatter.ex @@ -120,8 +120,6 @@ defmodule ExUnit.Formatter do * `:error_info` - Should format error information. - * `:error_info` - Should format error information. - * `:test_module_info` - Should format test module information. The message returned when this key is passed precedes messages such as `"failure on setup_all callback [...]"`. From 6c463b231078b420b70aa4be6fd58ca3518c74b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Mon, 4 Mar 2024 20:44:06 +0100 Subject: [PATCH 162/185] Ensure install-related functions do not crash when Mix is not started (#13391) --- lib/mix/lib/mix.ex | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index 7091a8b79a2..df2146e60b6 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -1047,7 +1047,7 @@ defmodule Mix do @doc false def in_install_project(fun) do - case Mix.State.get(:installed) do + case safe_get_installed() do {id, dynamic_config} -> config = install_project_config(dynamic_config) @@ -1075,7 +1075,7 @@ defmodule Mix do @doc since: "1.16.2" @spec install_project_dir() :: Path.t() | nil def install_project_dir() do - case Mix.State.get(:installed) do + case safe_get_installed() do {id, _dynamic_config} -> install_project_dir(id) nil -> nil end @@ -1086,7 +1086,17 @@ defmodule Mix do """ @doc since: "1.13.0" def installed? do - Mix.State.get(:installed) != nil + safe_get_installed() != nil + end + + defp safe_get_installed() do + if mix_started?() do + Mix.State.get(:installed) + end + end + + defp mix_started?() do + Process.whereis(Mix.State) != nil end defp stop_apps([]), do: :ok From f64ccd0871e389bee83bb22ef1c00d7f8f649d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 5 Mar 2024 18:37:05 +0100 Subject: [PATCH 163/185] Improve capture_log docs with latest Logger --- lib/ex_unit/lib/ex_unit/capture_log.ex | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/capture_log.ex b/lib/ex_unit/lib/ex_unit/capture_log.ex index 4d3ffdf2e3e..453973e1fba 100644 --- a/lib/ex_unit/lib/ex_unit/capture_log.ex +++ b/lib/ex_unit/lib/ex_unit/capture_log.ex @@ -42,16 +42,20 @@ defmodule ExUnit.CaptureLog do @doc """ Captures Logger messages generated when evaluating `fun`. - Returns the binary which is the captured output. + Returns the binary which is the captured output. The captured log + messages will be formatted using `Logger.default_formatter/1`. Any + option, besides the `:level`, will be forwarded as an override to + the default formatter. - This function mutes the `:console` backend and captures any log + This function mutes the default logger handler and captures any log messages sent to Logger from the calling processes. It is possible to ensure explicit log messages from other processes are captured by waiting for their exit or monitor signal. - Note that when the `async` is set to `true`, the messages from another - test might be captured. This is OK as long you consider such cases in - your assertions. + Note that when the `async` is set to `true` on `use ExUnit.Case`, + messages from other tests might be captured. This is OK as long + you consider such cases in your assertions, typically by using + the `=~/2` operator to perform partial matches. It is possible to configure the level to capture with `:level`, which will set the capturing level for the duration of the @@ -63,10 +67,6 @@ defmodule ExUnit.CaptureLog do configured in this function, no message will be captured. The behaviour is undetermined if async tests change Logger level. - The format, metadata and colors can be configured with `:format`, - `:metadata` and `:colors` respectively. These three options - defaults to the `:console` backend configuration parameters. - To get the result of the evaluation along with the captured log, use `with_log/2`. """ From 60e6ceae10b9a9743bf86f652ec1099264f470f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 6 Mar 2024 12:59:12 +0100 Subject: [PATCH 164/185] Correct task link in docs, closes #13388 --- lib/elixir/lib/task.ex | 2 +- lib/elixir/lib/task/supervisor.ex | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/elixir/lib/task.ex b/lib/elixir/lib/task.ex index 1b221050faa..7d5a7e4cc9d 100644 --- a/lib/elixir/lib/task.ex +++ b/lib/elixir/lib/task.ex @@ -40,7 +40,7 @@ defmodule Task do as they are *always* sent. If you are not expecting a reply, consider using `Task.start_link/1` as detailed below. - 2. async tasks link the caller and the spawned process. This + 2. Async tasks link the caller and the spawned process. This means that, if the caller crashes, the task will crash too and vice-versa. This is on purpose: if the process meant to receive the result no longer exists, there is diff --git a/lib/elixir/lib/task/supervisor.ex b/lib/elixir/lib/task/supervisor.ex index 7dacc2d6ffe..fff5f6a4717 100644 --- a/lib/elixir/lib/task/supervisor.ex +++ b/lib/elixir/lib/task/supervisor.ex @@ -152,7 +152,7 @@ defmodule Task.Supervisor do Starts a task that can be awaited on. The `supervisor` must be a reference as defined in `Supervisor`. - The task will still be linked to the caller, see `Task.async/3` for + The task will still be linked to the caller, see `Task.async/1` for more information and `async_nolink/3` for a non-linked variant. Raises an error if `supervisor` has reached the maximum number of @@ -174,7 +174,7 @@ defmodule Task.Supervisor do Starts a task that can be awaited on. The `supervisor` must be a reference as defined in `Supervisor`. - The task will still be linked to the caller, see `Task.async/3` for + The task will still be linked to the caller, see `Task.async/1` for more information and `async_nolink/3` for a non-linked variant. Raises an error if `supervisor` has reached the maximum number of @@ -196,7 +196,7 @@ defmodule Task.Supervisor do Starts a task that can be awaited on. The `supervisor` must be a reference as defined in `Supervisor`. - The task won't be linked to the caller, see `Task.async/3` for + The task won't be linked to the caller, see `Task.async/1` for more information. Raises an error if `supervisor` has reached the maximum number of @@ -283,7 +283,7 @@ defmodule Task.Supervisor do Starts a task that can be awaited on. The `supervisor` must be a reference as defined in `Supervisor`. - The task won't be linked to the caller, see `Task.async/3` for + The task won't be linked to the caller, see `Task.async/1` for more information. Raises an error if `supervisor` has reached the maximum number of From 17cd103a281899e1d132237183984859bbf020df Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Tue, 5 Mar 2024 22:31:05 +0900 Subject: [PATCH 165/185] Logger handles :process_label from OTP27 (#13392) --- lib/logger/lib/logger/translator.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/logger/lib/logger/translator.ex b/lib/logger/lib/logger/translator.ex index 51446bea505..f92fa407f59 100644 --- a/lib/logger/lib/logger/translator.ex +++ b/lib/logger/lib/logger/translator.ex @@ -404,11 +404,9 @@ defmodule Logger.Translator do end defp report_crash(min_level, crashed, extra, linked) do - [ - {:pid, pid}, - {:registered_name, name}, - {:error_info, {kind, reason, stack}} | crashed - ] = crashed + {pid, crashed} = Keyword.pop_first(crashed, :pid) + {name, crashed} = Keyword.pop_first(crashed, :registered_name) + {{kind, reason, stack}, crashed} = Keyword.pop_first(crashed, :error_info) dictionary = crashed[:dictionary] reason = Exception.normalize(kind, reason, stack) @@ -459,6 +457,10 @@ defmodule Logger.Translator do [prefix, "Ancestors: ", inspect(ancestors) | crash_info(min_level, debug, prefix)] end + defp crash_info(min_level, [{:process_label, _} | info], prefix) do + crash_info(min_level, info, prefix) + end + defp crash_info(:debug, debug, prefix) do for {key, value} <- debug do crash_debug(key, value, prefix) From ab1e3448f0236b8faabfbef7395c93b26c1b25ad Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Wed, 6 Mar 2024 19:00:32 +0900 Subject: [PATCH 166/185] Replace deprecated :code.lib_dir/2 usages (#13395) --- lib/elixir/lib/protocol.ex | 4 ++-- lib/elixir/test/elixir/protocol/consolidation_test.exs | 6 ++++-- lib/iex/test/iex/helpers_test.exs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/elixir/lib/protocol.ex b/lib/elixir/lib/protocol.ex index b1bc957a1b9..3b88d29811d 100644 --- a/lib/elixir/lib/protocol.ex +++ b/lib/elixir/lib/protocol.ex @@ -448,7 +448,7 @@ defmodule Protocol do ## Examples # Get Elixir's ebin directory path and retrieve all protocols - iex> path = :code.lib_dir(:elixir, :ebin) + iex> path = Application.app_dir(:elixir, "ebin") iex> mods = Protocol.extract_protocols([path]) iex> Enumerable in mods true @@ -477,7 +477,7 @@ defmodule Protocol do ## Examples # Get Elixir's ebin directory path and retrieve all protocols - iex> path = :code.lib_dir(:elixir, :ebin) + iex> path = Application.app_dir(:elixir, "ebin") iex> mods = Protocol.extract_impls(Enumerable, [path]) iex> List in mods true diff --git a/lib/elixir/test/elixir/protocol/consolidation_test.exs b/lib/elixir/test/elixir/protocol/consolidation_test.exs index 20309726cdf..4b4003e3276 100644 --- a/lib/elixir/test/elixir/protocol/consolidation_test.exs +++ b/lib/elixir/test/elixir/protocol/consolidation_test.exs @@ -161,13 +161,15 @@ defmodule Protocol.ConsolidationTest do end test "consolidation extracts protocols" do - protos = Protocol.extract_protocols([:code.lib_dir(:elixir, :ebin)]) + protos = Protocol.extract_protocols([Application.app_dir(:elixir, "ebin")]) assert Enumerable in protos assert Inspect in protos end test "consolidation extracts implementations with charlist path" do - protos = Protocol.extract_impls(Enumerable, [:code.lib_dir(:elixir, :ebin)]) + protos = + Protocol.extract_impls(Enumerable, [to_charlist(Application.app_dir(:elixir, "ebin"))]) + assert List in protos assert Function in protos end diff --git a/lib/iex/test/iex/helpers_test.exs b/lib/iex/test/iex/helpers_test.exs index d2580eeb297..18b19ddce56 100644 --- a/lib/iex/test/iex/helpers_test.exs +++ b/lib/iex/test/iex/helpers_test.exs @@ -144,7 +144,7 @@ defmodule IEx.HelpersTest do describe "open" do @iex_helpers "iex/lib/iex/helpers.ex" @elixir_erl "elixir/src/elixir.erl" - @lists_erl "#{:code.lib_dir(:stdlib, :src)}/lists.erl" + @lists_erl Application.app_dir(:stdlib, "src/lists.erl") @httpc_erl "src/http_client/httpc.erl" @editor System.get_env("ELIXIR_EDITOR") From 558e0ba6a6c0fcca066ec0d18cbad6b2615f2bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 1 Mar 2024 11:17:34 +0100 Subject: [PATCH 167/185] Discard mermaid fenced blocks from ansi docs --- lib/elixir/lib/io/ansi/docs.ex | 15 ++++++++++----- lib/elixir/test/elixir/io/ansi/docs_test.exs | 7 +++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/elixir/lib/io/ansi/docs.ex b/lib/elixir/lib/io/ansi/docs.ex index d0be03e3805..d76a48d13f9 100644 --- a/lib/elixir/lib/io/ansi/docs.ex +++ b/lib/elixir/lib/io/ansi/docs.ex @@ -176,12 +176,17 @@ defmodule IO.ANSI.Docs do process_code(rest, [line], indent, options) end - defp process(["```" <> _line | rest], text, indent, options) do - process_fenced_code_block(rest, text, indent, options, _delimiter = "```") + defp process(["```mermaid" <> _line | rest], text, indent, options) do + write_text(text, indent, options) + + rest + |> Enum.drop_while(&(&1 != "```")) + |> Enum.drop(1) + |> process([], indent, options) end - defp process(["~~~" <> _line | rest], text, indent, options) do - process_fenced_code_block(rest, text, indent, options, _delimiter = "~~~") + defp process(["```" <> _line | rest], text, indent, options) do + process_fenced_code_block(rest, text, indent, options, _delimiter = "```") end defp process(["