Looks like checkBundles from bug 1593276 doesn't make sure bundle was successful. This allowed for bug 1815494 after bug 1815224 landed.

https://treeherder.mozilla.org/jobs?repo=mozilla-central&searchStr=newtab&revision=b34aa450222401ca858232a109baae091f50c9d8&selectedTaskRun=Yyf8OU4FQieeTv5p4hZ3Xw.0

[task 2023-02-07T06:10:03.915Z] + node bin/try-runner.js
[task 2023-02-07T06:10:03.952Z] TEST START | checkBundles
[task 2023-02-07T06:10:04.848Z] [webpack-cli] Failed to load '/builds/worker/checkouts/gecko/browser/components/newtab/webpack.system-addon.config.js' config
[task 2023-02-07T06:10:04.848Z] [webpack-cli] Error: Cannot find module './resourceUriPlugin'
[task 2023-02-07T06:10:04.848Z] Require stack:
[task 2023-02-07T06:10:04.848Z] - /builds/worker/checkouts/gecko/browser/components/newtab/webpack.system-addon.config.js
[task 2023-02-07T06:10:04.848Z] - /builds/worker/checkouts/gecko/browser/components/newtab/node_modules/webpack-cli/lib/webpack-cli.js
[task 2023-02-07T06:10:04.848Z] - /builds/worker/checkouts/gecko/browser/components/newtab/node_modules/webpack-cli/lib/bootstrap.js
[task 2023-02-07T06:10:04.848Z] - /builds/worker/checkouts/gecko/browser/components/newtab/node_modules/webpack-cli/bin/cli.js
[task 2023-02-07T06:10:04.848Z]     at Function.Module._resolveFilename (node:internal/modules/cjs/loader:995:15)
[task 2023-02-07T06:10:04.848Z]     at Function.Module._load (node:internal/modules/cjs/loader:841:27)
[task 2023-02-07T06:10:04.848Z]     at Module.require (node:internal/modules/cjs/loader:1067:19)
[task 2023-02-07T06:10:04.848Z]     at require (node:internal/modules/cjs/helpers:103:18)
[task 2023-02-07T06:10:04.848Z]     at Object.<anonymous> (/builds/worker/checkouts/gecko/browser/components/newtab/webpack.system-addon.config.js:7:31)
[task 2023-02-07T06:10:04.848Z]     at Module._compile (node:internal/modules/cjs/loader:1165:14)
[task 2023-02-07T06:10:04.848Z]     at Object.Module._extensions..js (node:internal/modules/cjs/loader:1219:10)
[task 2023-02-07T06:10:04.848Z]     at Module.load (node:internal/modules/cjs/loader:1043:32)
[task 2023-02-07T06:10:04.848Z]     at Function.Module._load (node:internal/modules/cjs/loader:878:12)
[task 2023-02-07T06:10:04.848Z]     at Module.require (node:internal/modules/cjs/loader:1067:19) {
[task 2023-02-07T06:10:04.848Z]   code: 'MODULE_NOT_FOUND',
[task 2023-02-07T06:10:04.848Z]   requireStack: [
[task 2023-02-07T06:10:04.849Z]     '/builds/worker/checkouts/gecko/browser/components/newtab/webpack.system-addon.config.js',
[task 2023-02-07T06:10:04.849Z]     '/builds/worker/checkouts/gecko/browser/components/newtab/node_modules/webpack-cli/lib/webpack-cli.js',
[task 2023-02-07T06:10:04.849Z]     '/builds/worker/checkouts/gecko/browser/components/newtab/node_modules/webpack-cli/lib/bootstrap.js',
[task 2023-02-07T06:10:04.849Z]     '/builds/worker/checkouts/gecko/browser/components/newtab/node_modules/webpack-cli/bin/cli.js'
[task 2023-02-07T06:10:04.849Z]   ]
[task 2023-02-07T06:10:04.849Z] }
[task 2023-02-07T06:10:04.849Z] ERROR: "bundle:webpack" exited with 2.
[task 2023-02-07T06:10:04.852Z] TEST START | karma

https://hg.mozilla.org/mozilla-central/rev/eaf29b3a78e5