fix: remove the BOM from the compiled CSS

.changeset/remove-bom.md

@@ -0,0 +1,5 @@
+---
+"sass-loader": patch
+---
+
+Remove the byte order mark (BOM) from the compiled CSS. `dart-sass` prepends it when the compiled CSS contains non ASCII characters and the `style` option is `compressed` (the default for the `production` mode), but a BOM is only valid at the very beginning of a file - tools like `css-loader` move `@import` at-rules above it, so it ended up in the middle of the generated CSS and broke the rule after it. Source maps are shifted accordingly.

.cspell.json

@@ -18,7 +18,8 @@
     "commitlint",
     "bgcolor",
     "autocrlf",
-    "eslintcache"
+    "eslintcache",
+    "rspack"
   ],
   "ignorePaths": [
     "CHANGELOG.md",

src/index.js

@@ -9,6 +9,7 @@ import {
   getSassImplementation,
   getSassOptions,
   normalizeSourceMap,
+  removeBOM,
 } from "./utils.js";
 
 /** @typedef {import("webpack").LoaderContext<LoaderOptions>} LoaderContext */
@@ -89,6 +90,8 @@ async function loader(content) {
     map = normalizeSourceMap(map, this.rootContext);
   }
 
+  const { css, map: cssMap } = removeBOM(result.css.toString(), map);
+
   if (typeof result.loadedUrls !== "undefined") {
     for (const includedFile of result.loadedUrls.filter(
       (loadedUrl) => loadedUrl.protocol === "file:",
@@ -102,7 +105,7 @@ async function loader(content) {
     }
   }
 
-  callback(null, result.css.toString(), map || undefined);
+  callback(null, css, cssMap || undefined);
 }
 
 export default loader;

src/utils.js

@@ -764,6 +764,108 @@ function normalizeSourceMap(map, rootContext) {
   return newMap;
 }
 
+const BASE64_CHARACTERS =
+  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+const VLQ_BASE_SHIFT = 5;
+const VLQ_BASE = 1 << VLQ_BASE_SHIFT;
+const VLQ_BASE_MASK = VLQ_BASE - 1;
+const VLQ_CONTINUATION_BIT = VLQ_BASE;
+
+/**
+ * Decodes the first Base64 VLQ value of the given `mappings`.
+ * @param {string} mappings mappings of a source map
+ * @returns {{ value: number, length: number } | null} the decoded value and how many characters it takes, `null` when `mappings` doesn't start with a value
+ */
+function decodeVLQ(mappings) {
+  let value = 0;
+  let shift = 0;
+  let length = 0;
+  let digit;
+
+  do {
+    digit = BASE64_CHARACTERS.indexOf(mappings[length]);
+
+    if (digit === -1) {
+      return null;
+    }
+
+    value += (digit & VLQ_BASE_MASK) << shift;
+    shift += VLQ_BASE_SHIFT;
+    length += 1;
+  } while ((digit & VLQ_CONTINUATION_BIT) !== 0);
+
+  const isNegative = (value & 1) === 1;
+
+  value >>= 1;
+
+  return { value: isNegative ? -value : value, length };
+}
+
+/**
+ * @param {number} value value
+ * @returns {string} the Base64 VLQ encoded value
+ */
+function encodeVLQ(value) {
+  let encoded = "";
+  let vlq = value < 0 ? (-value << 1) | 1 : value << 1;
+
+  do {
+    let digit = vlq & VLQ_BASE_MASK;
+
+    vlq >>>= VLQ_BASE_SHIFT;
+
+    if (vlq > 0) {
+      digit |= VLQ_CONTINUATION_BIT;
+    }
+
+    encoded += BASE64_CHARACTERS[digit];
+  } while (vlq > 0);
+
+  return encoded;
+}
+
+/**
+ * Removes the byte order mark (BOM) from the compiled CSS.
+ *
+ * `dart-sass` prepends a BOM when the `charset` option is enabled (by default),
+ * the `style` option is `compressed` (the default for the `production` mode) and
+ * the compiled CSS contains non ASCII characters.
+ * A BOM is only meaningful at the very beginning of a file, but the loader result
+ * is just a string for webpack, and tools like `css-loader` move `@import` at-rules
+ * above it, so the BOM ends up in the middle of the generated CSS and breaks the
+ * first rule after it.
+ *
+ * Webpack removes a BOM a loader produced itself since
+ * https://github.com/webpack/webpack/pull/21857 and keeps the source map in sync
+ * with it since https://github.com/webpack/webpack/pull/21861, which makes this a
+ * no-op there. It is still needed for the webpack versions the `peerDependencies`
+ * range allows and for `rspack`, which passes a string result through untouched.
+ * Remove it once both handle this in the minimum version we support.
+ * @see https://github.com/webpack/sass-loader/issues/1335
+ * @param {string} css compiled CSS
+ * @param {RawSourceMap=} map source map
+ * @returns {{ css: string, map: RawSourceMap | undefined }} the CSS without the BOM and the source map adjusted to it
+ */
+function removeBOM(css, map) {
+  if (css.charCodeAt(0) !== 0xfe_ff) {
+    return { css, map };
+  }
+
+  // Sass counts the BOM as the first column of the first line, so all mappings
+  // of this line need to be shifted by one column.
+  // Only the first segment has to be updated - the generated column of the
+  // segments after it is relative to the previous one.
+  if (map && typeof map.mappings === "string") {
+    const decoded = decodeVLQ(map.mappings);
+
+    if (decoded && decoded.value > 0) {
+      map.mappings = `${encodeVLQ(decoded.value - 1)}${map.mappings.slice(decoded.length)}`;
+    }
+  }
+
+  return { css: css.slice(1), map };
+}
+
 /**
  * @param {Error | SassError} error the original sass error
  * @returns {Error} a new error
@@ -790,4 +892,5 @@ export {
   getSassOptions,
   getWebpackResolver,
   normalizeSourceMap,
+  removeBOM,
 };

test/__snapshots__/loader.test.js.snap

@@ -1446,6 +1446,102 @@ exports[`loader > should prefer relative import ('sass-embedded', 'modern-compil
 []
 `;
 
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern' API, 'sass' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern' API, 'sass' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern' API, 'sass' syntax) 3`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern' API, 'scss' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern' API, 'scss' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern' API, 'scss' syntax) 3`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern-compiler' API, 'sass' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern-compiler' API, 'sass' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern-compiler' API, 'sass' syntax) 3`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern-compiler' API, 'scss' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern-compiler' API, 'scss' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('dart-sass', 'modern-compiler' API, 'scss' syntax) 3`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern' API, 'sass' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern' API, 'sass' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern' API, 'sass' syntax) 3`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern' API, 'scss' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern' API, 'scss' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern' API, 'scss' syntax) 3`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern-compiler' API, 'sass' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern-compiler' API, 'sass' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern-compiler' API, 'sass' syntax) 3`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern-compiler' API, 'scss' syntax) 1`] = `
+"p{content:\\"é\\"}"
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern-compiler' API, 'scss' syntax) 2`] = `
+[]
+`;
+
+exports[`loader > should remove the BOM from the compiled CSS ('sass-embedded', 'modern-compiler' API, 'scss' syntax) 3`] = `
+[]
+`;
+
 exports[`loader > should resolve absolute paths ('dart-sass', 'modern' API, 'sass' syntax) 1`] = `
 "@charset \\"UTF-8\\";\\n@import \\"./file.css\\";\\nbody {\\n  font: 100% Helvetica, sans-serif;\\n  color: #333;\\n}\\n\\nnav ul {\\n  margin: 0;\\n  padding: 0;\\n  list-style: none;\\n}\\nnav li {\\n  display: inline-block;\\n}\\nnav a {\\n  display: block;\\n  padding: 6px 12px;\\n  text-decoration: none;\\n}\\n\\n.box {\\n  -webkit-border-radius: 10px;\\n  -moz-border-radius: 10px;\\n  -ms-border-radius: 10px;\\n  border-radius: 10px;\\n}\\n\\n.message, .warning, .error, .success {\\n  border: 1px solid #ccc;\\n  padding: 10px;\\n  color: #333;\\n}\\n\\n.success {\\n  border-color: green;\\n}\\n\\n.error {\\n  border-color: red;\\n}\\n\\n.warning {\\n  border-color: yellow;\\n}\\n\\n.foo:before {\\n  content: \\"\\\\e0c6\\";\\n}\\n\\n.bar:before {\\n  content: \\"∑\\";\\n}"
 `;
@@ -3327,7 +3423,7 @@ exports[`loader > should work and output deprecation message (sass-embedded) 9`]
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern' API, 'sass' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern' API, 'sass' syntax) 2`] = `
@@ -3339,7 +3435,7 @@ exports[`loader > should work and output the \"compressed\" outputStyle when \"m
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern' API, 'scss' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern' API, 'scss' syntax) 2`] = `
@@ -3351,7 +3447,7 @@ exports[`loader > should work and output the \"compressed\" outputStyle when \"m
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern-compiler' API, 'sass' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern-compiler' API, 'sass' syntax) 2`] = `
@@ -3363,7 +3459,7 @@ exports[`loader > should work and output the \"compressed\" outputStyle when \"m
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern-compiler' API, 'scss' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`loader > should work and output the \"compressed\" outputStyle when \"mode\" is production ('dart-sass', 'modern-compiler' API, 'scss' syntax) 2`] = `

test/__snapshots__/sassOptions-option.test.js.snap

@@ -287,7 +287,7 @@ exports[`sassOptions option > should respect the \"style\" option ('sass-embedde
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern' API, 'sass' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern' API, 'sass' syntax) 2`] = `
@@ -299,7 +299,7 @@ exports[`sassOptions option > should use \"compressed\" output style in the \"pr
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern' API, 'scss' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern' API, 'scss' syntax) 2`] = `
@@ -311,7 +311,7 @@ exports[`sassOptions option > should use \"compressed\" output style in the \"pr
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern-compiler' API, 'sass' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.message,.warning,.error,.success{border:1px solid #ccc;padding:10px;color:#333}.success{border-color:green}.error{border-color:red}.warning{border-color:#ff0}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern-compiler' API, 'sass' syntax) 2`] = `
@@ -323,7 +323,7 @@ exports[`sassOptions option > should use \"compressed\" output style in the \"pr
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern-compiler' API, 'scss' syntax) 1`] = `
-"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
+"@import\\"./file.css\\";body{font:100% Helvetica,sans-serif;color:#333}nav ul{margin:0;padding:0;list-style:none}nav li{display:inline-block}nav a{display:block;padding:6px 12px;text-decoration:none}.box{-webkit-border-radius:10px;-moz-border-radius:10px;-ms-border-radius:10px;border-radius:10px}.foo:before{content:\\"\\"}.bar:before{content:\\"∑\\"}"
 `;
 
 exports[`sassOptions option > should use \"compressed\" output style in the \"production\" mode ('dart-sass', 'modern-compiler' API, 'scss' syntax) 2`] = `