Respect webpackIgnore and turboIgnore comments in the parser (#68451)

## What This is step one of PACK-3046, setting up the necessary parser-side changes to detect and store the webpackIgnore and turbopackIgnore comment directives. This is followed by another PR that implements the bundler side. ## Why Webpack supports special directives to prevent bundle-time linking of specific imports. Some libraries need this (mapbox) and so we should support it. ## How The directives are stored in comments within `require` and `import` calls. We parse the comments alongside traversal and look for comments that exist inside the function calls. These are then collected and used in later stages when performing linking to opt out of linking packages that have been explicitly excluded.
vercel · Aug 27, 2024 · ab7744e · ab7744e
1 parent 678b487
commit ab7744e
Show file tree

Hide file tree

Showing 16 changed files with 961 additions and 17 deletions.
diff --git a/turbopack/crates/turbopack-ecmascript/benches/analyzer.rs b/turbopack/crates/turbopack-ecmascript/benches/analyzer.rs
@@ -56,7 +56,7 @@ pub fn benchmark(c: &mut Criterion) {
                 program.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
 
                 let eval_context =
-                    EvalContext::new(&program, unresolved_mark, top_level_mark, None);
+                    EvalContext::new(&program, unresolved_mark, top_level_mark, None, None);
                 let var_graph = create_graph(&program, &eval_context);
 
                 let input = BenchInput {

diff --git a/turbopack/crates/turbopack-ecmascript/readme.md b/turbopack/crates/turbopack-ecmascript/readme.md
@@ -0,0 +1,14 @@
+# turbopack-ecmascript
+
+## Adding new parser tests
+
+We use a snapshot-based testing system to ensure that changes to the parser don't break existing code.
+To add a new test, you need to create a new directory in `tests/analyzer/graph` with an 'input.js' file
+inside.
+
+The snapshot tests are done with the `testing` crate. You can upate them by passing the env var
+`UPDATE=1` to the test runner.
+
+```sh
+UPDATE=1 cargo test -p turbopack-ecmascript
+```
diff --git a/turbopack/crates/turbopack-ecmascript/src/analyzer/graph.rs b/turbopack/crates/turbopack-ecmascript/src/analyzer/graph.rs
@@ -6,7 +6,7 @@ use std::{
 
 use swc_core::{
     atoms::Atom,
-    common::{pass::AstNodePath, Mark, Span, Spanned, SyntaxContext, GLOBALS},
+    common::{comments::Comments, pass::AstNodePath, Mark, Span, Spanned, SyntaxContext, GLOBALS},
     ecma::{
         ast::*,
         atoms::js_word,
@@ -294,23 +294,29 @@ pub fn create_graph(m: &Program, eval_context: &EvalContext) -> VarGraph {
     graph
 }
 
+/// A context used for assembling the evaluation graph.
+#[derive(Debug)]
 pub struct EvalContext {
     pub(crate) unresolved_mark: Mark,
     pub(crate) top_level_mark: Mark,
     pub(crate) imports: ImportMap,
 }
 
 impl EvalContext {
+    /// Produce a new [EvalContext] from a [Program]. If you wish to support
+    /// webpackIgnore or turbopackIgnore comments, you must pass those in,
+    /// since the AST does not include comments by default.
     pub fn new(
         module: &Program,
         unresolved_mark: Mark,
         top_level_mark: Mark,
+        comments: Option<&dyn Comments>,
         source: Option<Vc<Box<dyn Source>>>,
     ) -> Self {
         Self {
             unresolved_mark,
             top_level_mark,
-            imports: ImportMap::analyze(module, source),
+            imports: ImportMap::analyze(module, source, comments),
         }
     }
 

diff --git a/turbopack/crates/turbopack-ecmascript/src/analyzer/imports.rs b/turbopack/crates/turbopack-ecmascript/src/analyzer/imports.rs
@@ -1,9 +1,12 @@
-use std::{collections::BTreeMap, fmt::Display};
+use std::{
+    collections::{BTreeMap, HashMap},
+    fmt::Display,
+};
 
 use indexmap::{IndexMap, IndexSet};
 use once_cell::sync::Lazy;
 use swc_core::{
-    common::{source_map::SmallPos, Span},
+    common::{comments::Comments, source_map::SmallPos, Span, Spanned},
     ecma::{
         ast::*,
         atoms::{js_word, JsWord},
@@ -136,6 +139,17 @@ pub(crate) struct ImportMap {
 
     /// True if the module is an ESM module due to top-level await.
     has_top_level_await: bool,
+
+    /// Locations of webpackIgnore or turbopackIgnore comments
+    /// This is a webpack feature that allows opting out of static
+    /// imports, which we should respect.
+    ///
+    /// Example:
+    /// ```js
+    /// const a = import(/* webpackIgnore: true */ "a");
+    /// const b = import(/* turbopackIgnore: true */ "b");
+    /// ```
+    turbopack_ignores: HashMap<Span, bool>,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -200,12 +214,17 @@ impl ImportMap {
     }
 
     /// Analyze ES import
-    pub(super) fn analyze(m: &Program, source: Option<Vc<Box<dyn Source>>>) -> Self {
+    pub(super) fn analyze(
+        m: &Program,
+        source: Option<Vc<Box<dyn Source>>>,
+        comments: Option<&dyn Comments>,
+    ) -> Self {
         let mut data = ImportMap::default();
 
         m.visit_with(&mut Analyzer {
             data: &mut data,
             source,
+            comments,
         });
 
         data
@@ -215,6 +234,7 @@ impl ImportMap {
 struct Analyzer<'a> {
     data: &'a mut ImportMap,
     source: Option<Vc<Box<dyn Source>>>,
+    comments: Option<&'a dyn Comments>,
 }
 
 impl<'a> Analyzer<'a> {
@@ -409,8 +429,63 @@ impl Visit for Analyzer<'_> {
     fn visit_export_default_expr(&mut self, _: &ExportDefaultExpr) {
         self.data.has_exports = true;
     }
-    fn visit_stmt(&mut self, _: &Stmt) {
-        // don't visit children
+
+    fn visit_stmt(&mut self, n: &Stmt) {
+        if self.comments.is_some() {
+            // only visit children if we potentially need to mark import / requires
+            n.visit_children_with(self);
+        }
+    }
+
+    /// check if import or require contains an ignore comment
+    ///
+    /// We are checking for the following cases:
+    /// - import(/* webpackIgnore: true */ "a")
+    /// - require(/* webpackIgnore: true */ "a")
+    ///
+    /// We can do this by checking if any of the comment spans are between the
+    /// callee and the first argument.
+    fn visit_call_expr(&mut self, n: &CallExpr) {
+        // we can actually unwrap thanks to the optimisation above
+        // but it can't hurt to be safe...
+        if let Some(comments) = self.comments {
+            let callee_span = match &n.callee {
+                Callee::Import(Import { span, .. }) => Some(span),
+                // this assumes you cannot reassign `require`
+                Callee::Expr(box Expr::Ident(Ident { span, sym, .. })) if sym == "require" => {
+                    Some(span)
+                }
+                _ => None,
+            };
+
+            // we are interested here in the last comment with a valid directive
+            let ignore_statement = n
+                .args
+                .first()
+                .map(|arg| arg.span_lo())
+                .and_then(|comment_pos| comments.get_leading(comment_pos))
+                .iter()
+                .flatten()
+                .rev()
+                .filter_map(|comment| {
+                    let (directive, value) = comment.text.trim().split_once(':')?;
+                    // support whitespace between the colon
+                    match (directive.trim(), value.trim()) {
+                        ("webpackIgnore" | "turbopackIgnore", "true") => Some(true),
+                        ("webpackIgnore" | "turbopackIgnore", "false") => Some(false),
+                        _ => None, // ignore anything else
+                    }
+                })
+                .next();
+
+            if let Some((callee_span, ignore_statement)) = callee_span.zip(ignore_statement) {
+                self.data
+                    .turbopack_ignores
+                    .insert(*callee_span, ignore_statement);
+            };
+        }
+
+        n.visit_children_with(self);
     }
 
     fn visit_program(&mut self, m: &Program) {

diff --git a/turbopack/crates/turbopack-ecmascript/src/analyzer/mod.rs b/turbopack/crates/turbopack-ecmascript/src/analyzer/mod.rs
@@ -3822,7 +3822,8 @@ mod tests {
                 let top_level_mark = Mark::new();
                 m.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
 
-                let eval_context = EvalContext::new(&m, unresolved_mark, top_level_mark, None);
+                let eval_context =
+                    EvalContext::new(&m, unresolved_mark, top_level_mark, None, None);
 
                 let mut var_graph = create_graph(&m, &eval_context);
 

diff --git a/turbopack/crates/turbopack-ecmascript/src/lib.rs b/turbopack/crates/turbopack-ecmascript/src/lib.rs
@@ -366,7 +366,7 @@ impl EcmascriptParsable for EcmascriptModuleAsset {
 impl EcmascriptAnalyzable for EcmascriptModuleAsset {
     #[turbo_tasks::function]
     fn analyze(self: Vc<Self>) -> Vc<AnalyzeEcmascriptModuleResult> {
-        analyse_ecmascript_module(self, None)
+        analyse_ecmascript_module(self, None, None)
     }
 
     /// Generates module contents without an analysis pass. This is useful for

diff --git a/turbopack/crates/turbopack-ecmascript/src/parse.rs b/turbopack/crates/turbopack-ecmascript/src/parse.rs
@@ -433,6 +433,7 @@ async fn parse_content(
                 &parsed_program,
                 unresolved_mark,
                 top_level_mark,
+                None,
                 Some(source),
             );
 

diff --git a/turbopack/crates/turbopack-ecmascript/src/references/mod.rs b/turbopack/crates/turbopack-ecmascript/src/references/mod.rs
@@ -17,7 +17,7 @@ pub mod util;
 
 use std::{
     borrow::Cow,
-    collections::{BTreeMap, HashMap},
+    collections::{BTreeMap, HashMap, HashSet},
     future::Future,
     mem::take,
     pin::Pin,
@@ -393,16 +393,23 @@ where
     HANDLER.set(handler, || GLOBALS.set(globals, f))
 }
 
+/// Analyse a provided [EcmascriptModuleAsset] and return a
+/// [AnalyzeEcmascriptModuleResult].
+///
+/// # Arguments
+/// * ignored_spans - A set of spans to ignore when analysing the module. This is useful for example
+///   to respect turbopackIgnore directives on ignores.
 #[turbo_tasks::function]
 pub(crate) async fn analyse_ecmascript_module(
     module: Vc<EcmascriptModuleAsset>,
     part: Option<Vc<ModulePart>>,
+    ignored_spans: Option<Vc<HashSet<Span>>>,
 ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
     let span = {
         let module = module.ident().to_string().await?.to_string();
         tracing::info_span!("analyse ecmascript module", module = module)
     };
-    let result = analyse_ecmascript_module_internal(module, part)
+    let result = analyse_ecmascript_module_internal(module, part, ignored_spans)
         .instrument(span)
         .await;
 
@@ -418,6 +425,7 @@ pub(crate) async fn analyse_ecmascript_module(
 pub(crate) async fn analyse_ecmascript_module_internal(
     module: Vc<EcmascriptModuleAsset>,
     part: Option<Vc<ModulePart>>,
+    _webpack_ignored_effects: Option<Vc<HashSet<Span>>>,
 ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
     let raw_module = module.await?;
 

diff --git a/turbopack/crates/turbopack-ecmascript/src/tree_shake/asset.rs b/turbopack/crates/turbopack-ecmascript/src/tree_shake/asset.rs
@@ -1,4 +1,7 @@
+use std::collections::HashSet;
+
 use anyhow::{Context, Result};
+use swc_core::common::Span;
 use turbo_tasks::Vc;
 use turbopack_core::{
     asset::{Asset, AssetContent},
@@ -59,7 +62,11 @@ impl EcmascriptAnalyzable for EcmascriptModulePartAsset {
     async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
         let this = self.await?;
         let part = this.part;
-        Ok(analyse_ecmascript_module(this.full_module, Some(part)))
+        Ok(analyse_ecmascript_module(
+            this.full_module,
+            Some(part),
+            None,
+        ))
     }
 
     #[turbo_tasks::function]
@@ -131,7 +138,7 @@ impl Module for EcmascriptModulePartAsset {
     async fn references(&self) -> Result<Vc<ModuleReferences>> {
         let split_data = split_module(self.full_module).await?;
 
-        let analyze = analyze(self.full_module, self.part).await?;
+        let analyze = analyze(self.full_module, self.part, None).await?;
 
         let (deps, entrypoints) = match &*split_data {
             SplitResult::Ok {
@@ -264,16 +271,17 @@ impl EcmascriptModulePartAsset {
     pub(super) async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
         let this = self.await?;
 
-        Ok(analyze(this.full_module, this.part))
+        Ok(analyze(this.full_module, this.part, None))
     }
 }
 
 #[turbo_tasks::function]
-async fn analyze(
+fn analyze(
     module: Vc<EcmascriptModuleAsset>,
     part: Vc<ModulePart>,
+    ignored_spans: Option<Vc<HashSet<Span>>>,
 ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
-    Ok(analyse_ecmascript_module(module, Some(part)))
+    Ok(analyse_ecmascript_module(module, Some(part), ignored_spans))
 }
 
 #[turbo_tasks::value_impl]

diff --git a/turbopack/crates/turbopack-ecmascript/src/tree_shake/mod.rs b/turbopack/crates/turbopack-ecmascript/src/tree_shake/mod.rs
@@ -505,6 +505,7 @@ pub(super) async fn split(
                         &program,
                         eval_context.unresolved_mark,
                         eval_context.top_level_mark,
+                        None,
                         Some(source),
                     );
 
@@ -629,6 +630,7 @@ pub(super) async fn part_of_module(
                         eval_context.unresolved_mark,
                         eval_context.top_level_mark,
                         None,
+                        None,
                     );
 
                     return Ok(ParseResult::Ok {
@@ -706,6 +708,7 @@ pub(super) async fn part_of_module(
                         eval_context.unresolved_mark,
                         eval_context.top_level_mark,
                         None,
+                        None,
                     );
                     return Ok(ParseResult::Ok {
                         program,